## Lotka-Volterra

function lotka(du,u,p,t)
  x = u[1]
  y = u[2]
  du[1] = p[1]*x - p[2]*x*y
  du[2] = -p[3]*y + p[4]*x*y
end

"""
Lotka-Voltera Equations (Non-stiff)

```math
\\frac{dx}{dt} = ax - bxy
```
```math
\\frac{dy}{dt} = -cy + dxy
```

with initial condition ``x=y=1``
"""
prob_ode_lotkavoltera = ODEProblem(lotka,[1.0,1.0],(0.0,1.0),[1.5,1.0,3.0,1.0])

## Fitzhugh-Nagumo

function fitz(du,u,p,t)
  v = u[1]
  w = u[2]
  a = p[1]
  b = p[2]
  τinv = p[3]
  l = p[4]
  du[1] = v - v^3/3 -w + l
  du[2] = τinv*(v +  a - b*w)
end
"""
Fitzhugh-Nagumo (Non-stiff)

```math
\\frac{dv}{dt} = v - \\frac{v^3}{3} - w + I_{est}
```
```math
τ \\frac{dw}{dt} = v + a -bw
```

with initial condition ``v=w=1``
"""
prob_ode_fitzhughnagumo = ODEProblem(fitz,[1.0;1.0],(0.0,1.0),(0.7,0.8,1/12.5,0.5))

#Van der Pol Equations
@parameters t μ
@variables x(t) y(t)
@derivatives D'~t
eqs = [D(x) ~ μ*((1-x^2)*y - x),
       D(y) ~ y]
de = ODESystem(eqs)
van = ODEFunction(de, [x,y], [μ], jac=true, Wfact=true)

"""
Van der Pol Equations

```math
\\frac{dx}{dt} = y
```
```math
\\frac{dy}{dt} = μ((1-x^2)y -x)
```

with ``μ=1.0`` and ``u_0=[0,\\sqrt{3}]``

Non-stiff parameters.
"""
prob_ode_vanderpol = ODEProblem(van,[0;sqrt(3)],(0.0,1.0),1.0)

"""
Van der Pol Equations

```math
\\frac{dx}{dt} = y
```
```math
\\frac{dy}{dt} = μ(1-x^2)y -x
```

with ``μ=10^6`` and ``u_0=[0,\\sqrt{3}]``

Stiff parameters.
"""
prob_ode_vanstiff = ODEProblem(van,[0;sqrt(3)],(0.0,1.0),1e6)

# ROBER
@parameters t k₁ k₂ k₃
@variables y₁(t) y₂(t) y₃(t)
@derivatives D'~t
eqs = [D(y₁) ~ -k₁*y₁+k₃*y₂*y₃,
       D(y₂) ~ k₁*y₁-k₂*y₂^2-k₃*y₂*y₃,
       D(y₃) ~ k₂*y₂^2]
de = ODESystem(eqs)
rober = ODEFunction(de, [y₁,y₂,y₃], [k₁,k₂,k₃], jac=true, Wfact=true)

"""
The Robertson biochemical reactions: (Stiff)

```math
\\frac{dy₁}{dt} = -k₁y₁+k₃y₂y₃
```
```math
\\frac{dy₂}{dt} =  k₁y₁-k₂y₂^2-k₃y₂y₃
```
```math
\\frac{dy₃}{dt} =  k₂y₂^2
```

where ``k₁=0.04``, ``k₂=3\\times10^7``, ``k₃=10^4``. For details, see:

Hairer Norsett Wanner Solving Ordinary Differential Equations I - Nonstiff Problems Page 129

Usually solved on ``[0,1e11]``
"""
prob_ode_rober = ODEProblem(rober,[1.0;0.0;0.0],(0.0,1e11),(0.04,3e7,1e4))

# Three Body
const threebody_μ = big(0.012277471); const threebody_μ′ = 1 - threebody_μ

threebody = (du,u,p,t) -> begin
  # 1 = y₁
  # 2 = y₂
  # 3 = y₁'
  # 4 = y₂'
  D₁ = ((u[1]+threebody_μ)^2 + u[2]^2)^(3/2)
  D₂ = ((u[1]-threebody_μ′)^2 + u[2]^2)^(3/2)
  du[1] = u[3]
  du[2] = u[4]
  du[3] = u[1] + 2u[4] - threebody_μ′*(u[1]+threebody_μ)/D₁ - threebody_μ*(u[1]-threebody_μ′)/D₂
  du[4] = u[2] - 2u[3] - threebody_μ′*u[2]/D₁ - threebody_μ*u[2]/D₂
end

@doc doc"""
The ThreeBody problem as written by Hairer: (Non-stiff)

```math
\frac{dy₁}{dt} = y₁ + 2\frac{dy₂}{dt} - \bar{μ}\frac{y₁+μ}{D₁} - μ\frac{y₁-\bar{μ}}{D₂}
```
```math
\frac{dy₂}{dt} = y₂ - 2\frac{dy₁}{dt} - \bar{μ}\frac{y₂}{D₁} - μ\frac{y₂}{D₂}
```
```math
D₁ = ((y₁+μ)^2 + y₂^2)^{3/2}
```
```math
D₂ = ((y₁-\bar{μ})^2+y₂^2)^{3/2}
```
```math
μ = 0.012277471
```
```math
\bar{μ} =1-μ
```

From Hairer Norsett Wanner Solving Ordinary Differential Equations I - Nonstiff Problems Page 129

Usually solved on ``t₀ = 0.0`` and ``T = 17.0652165601579625588917206249``
Periodic with that setup.
"""
prob_ode_threebody = ODEProblem(threebody,[0.994, 0.0, 0.0, big(-2.00158510637908252240537862224)],(big(0.0),big(17.0652165601579625588917206249)))

# Rigid Body Equations

@parameters t I₁ I₂ I₃
@variables y₁(t) y₂(t) y₃(t)
@derivatives D'~t
eqs = [D(y₁) ~ I₁*y₂*y₃,
       D(y₂) ~ I₂*y₁*y₃,
       D(y₃) ~ I₃*y₁*y₂]
de = ODESystem(eqs)
rigid = ODEFunction(de, [y₁,y₂,y₃], [I₁,I₂,I₃], jac=true, Wfact=true)

"""
Rigid Body Equations (Non-stiff)

```math
\\frac{dy₁}{dt}  = I₁y₂y₃
```
```math
\\frac{dy₂}{dt}  = I₂y₁y₃
```
```math
\\frac{dy₃}{dt}  = I₃y₁y₂
```

with ``I₁=-2``, ``I₂=1.25``, and ``I₃=-1/2``.

The initial condition is ``y=[1.0;0.0;0.9]``.

From Solving Differential Equations in R by Karline Soetaert

or Hairer Norsett Wanner Solving Ordinary Differential Equations I - Nonstiff Problems Page 244

Usually solved from 0 to 20.
"""
prob_ode_rigidbody = ODEProblem(rigid,[1.0,0.0,0.9],(0.0,20.0),(-2.0,1.25,-0.5))

# Pleiades Problem

pleiades = (du,u,p,t) -> begin
  x = view(u,1:7)   # x
  y = view(u,8:14)  # y
  v = view(u,15:21) # x′
  w = view(u,22:28) # y′
  du[1:7] .= v
  du[8:14].= w
  for i in 14:28
    du[i] = zero(eltype(u))
  end
  for i=1:7,j=1:7
    if i != j
      r = ((x[i]-x[j])^2 + (y[i] - y[j])^2)^(3/2)
      du[14+i] += j*(x[j] - x[i])/r
      du[21+i] += j*(y[j] - y[i])/r
    end
  end
end

@doc doc"""
Pleiades Problem (Non-stiff)

```math
\frac{d^2xᵢ}{dt^2} = \sum_{j≠i} mⱼ(xⱼ-xᵢ)/rᵢⱼ
```
```math
\frac{d^2yᵢ}{dt^2} = \sum_{j≠i} mⱼ(yⱼ-yᵢ)/rᵢⱼ
```

where

```math
rᵢⱼ = ((xᵢ-xⱼ)^2 + (yᵢ-yⱼ)^2)^{3/2}
```

and initial conditions are

```math
x₁(0) = 3
```
```math
x₂(0) = 3
```
```math
x₃(0) = -1
```
```math
x₄(0) = -3
```
```math
x₅(0) = 2
```
```math
x₆(0) = -2
```
```math
x₇(0) = 2
```
```math
y₁(0) = 3
```
```math
y₂(0) = -3
```
```math
y₃(0) = 2
```
```math
y₄(0) = 0
```
```math
y₅(0) = 0
```
```math
y₆(0) = -4
```
```math
y₇(0) = 4
```

and with ``\frac{dxᵢ(0)}{dt}=\frac{dyᵢ(0)}{dt}=0`` except for

```math
\frac{dx₆(0)}{dt} = 1.75
```
```math
\frac{dx₇(0)}{dt} = -1.5
```
```math
\frac{dy₄(0)}{dt} = -1.25
```
```math
\frac{dy₅(0)}{dt} = 1
```

From Hairer Norsett Wanner Solving Ordinary Differential Equations I - Nonstiff Problems Page 244

Usually solved from 0 to 3.
"""
prob_ode_pleiades = ODEProblem(pleiades,[3.0,3.0,-1.0,-3.0,2.0,-2.0,2.0,3.0,-3.0,2.0,0,0,-4.0,4.0,0,0,0,0,0,1.75,-1.5,0,0,0,-1.25,1,0,0],(0.0,3.0))

Random.seed!(100)
const mm_A = rand(4,4)
mm_linear = function (du,u,p,t)
  mul!(du,mm_A,u)
end
const MM_linear =Matrix(Diagonal(0.5ones(4)))
mm_f = ODEFunction(mm_linear;analytic = (u0,p,t) -> exp(inv(MM_linear)*mm_A*t)*u0, mass_matrix=MM_linear)
prob_ode_mm_linear = ODEProblem(mm_f,rand(4),(0.0,1.0))

@parameters t p1 p2 p3 p4 p5 p6 p7 p8 p9 p10 p11 p12
@variables y1(t) y2(t) y3(t) y4(t) y5(t) y6(t) y7(t) y8(t)
@derivatives D'~t
eqs = [D(y1) ~ -p1*y1 + p2*y2 + p3*y3 + p4,
       D(y2) ~ p1*y1 - p5*y2,
       D(y3) ~ -p6*y3 + p2*y4 + p7*y5,
       D(y4) ~ p3*y2 + p1*y3 - p8*y4,
       D(y5) ~ -p9*y5 + p2*y6 + p2*y7,
       D(y6) ~ -p10*y6*y8 + p11*y4 + p1*y5 -
                p2*y6 + p11*y7,
       D(y7) ~  p10*y6*y8 - p12*y7,
       D(y8) ~ -p10*y6*y8 + p12*y7]
de = ODESystem(eqs)
hires = ODEFunction(de, [y1,y2,y3,y4,y5,y6,y7,y8],
                        [p1,p2,p3,p4,p5,p6,p7,p8,p9,p10,p11,p12],
                        jac=true, Wfact=true)

u0 = zeros(8)
u0[1] = 1
u0[8] = 0.0057

"""
Hires Problem (Stiff)

It is in the form of

```math
\\frac{dy}{dt} = f(y)
```

 with

```math
 y(0)=y_0, \\quad y \\in ℝ^8, \\quad 0 ≤ t ≤ 321.8122
```

where ``f`` is defined by

``f(y) = \\begin{pmatrix} −1.71y_1 & +0.43y_2 & +8.32y_3 & +0.0007y_4 & \\\\ 1.71y_1 & −8.75y_2 & & & \\\\ −10.03y_3 & +0.43y_4 & +0.035y_5 & & \\\\ 8.32y_2 & +1.71y_3 & −1.12y_4 & & \\\\ −1.745y_5 & +0.43y_6 & +0.43y_7 & & \\\\ −280y_6y_8 & +0.69y_4 & +1.71y_5 & −0.43y_6 & +0.69y_7 \\\\ 280y_6y_8 & −1.81y_7 & & & \\\\ −280y_6y_8 & +1.81y_7 & & & \\end{pmatrix}``

Reference: [demohires.pdf](http://www.radford.edu/~thompson/vodef90web/problems/demosnodislin/Demos_Pitagora/DemoHires/demohires.pdf)
Notebook: [Hires.ipynb](http://nbviewer.jupyter.org/github/JuliaDiffEq/DiffEqBenchmarks.jl/blob/master/StiffODE/Hires.ipynb)
"""
prob_ode_hires = ODEProblem(hires,u0,(0.0,321.8122), (1.71, 0.43, 8.32, 0.0007, 8.75,
                                                      10.03, 0.035, 1.12, 1.745, 280.0,
                                                      0.69, 1.81))

@parameters t p1 p2 p3
@variables y1(t) y2(t) y3(t)
@derivatives D'~t
eqs = [D(y1) ~ p1*(y2+y1*(1-p2*y1-y2)),
       D(y2) ~ (y3-(1+y1)*y2)/p1,
       D(y3) ~ p3*(y1-y3)]
de = ODESystem(eqs)
jac = calculate_jacobian(de)
ModelingToolkit.calculate_factorized_W(de)
orego = ODEFunction(de, [y1,y2,y3], [p1,p2,p3], jac=true, Wfact=true)

"""
Orego Problem (Stiff)

It is in the form of ``\\frac{dy}{dt}=f(y), \\quad y(0)=y0,`` with

```math
y \\in ℝ^3, \\quad 0 ≤ t ≤ 360
```

where ``f`` is defined by

``f(y) = \\begin{pmatrix} s(y_2 - y_1(1-qy_1-y_2)) \\\\ (y_3 - y_2(1+y_1))/s \\\\ w(y_1-y_3) \\end{pmatrix}``

where ``s=77.27``, ``w=0.161`` and ``q=8.375⋅10^{-6}``.

Reference: [demoorego.pdf](http://www.radford.edu/~thompson/vodef90web/problems/demosnodislin/Demos_Pitagora/DemoOrego/demoorego.pdf)
Notebook: [Orego.ipynb](http://nbviewer.jupyter.org/github/JuliaDiffEq/DiffEqBenchmarks.jl/blob/master/StiffODE/Orego.ipynb)
"""
prob_ode_orego = ODEProblem(orego,[1.0,2.0,3.0],(0.0,30.0),[77.27,8.375e-6,0.161])


# # Cellular Automaton
#
# ## Abstract
# We have an arrangement of atoms/cells. Each cell belongs to one in a finite
# set of states. With the progression of a unit time cells change their state
# in a parallel fashion determined by the rules of the system.
#
# Depending on the initial state and thr rules of the system we get a variety
# of behaviours. Some systems lead to homogenous states, some exhibit periodic
# oscillations between states, some are chaotic non-deterministic behaviours,
# ,.etc . The study of these discrete and abstract computational systems is
# called cellular automaton.
# ![https://plato.stanford.edu/entries/cellular-automata/](https://plato.stanford.edu/entries/cellular-automata/)
# is a great resource to understand and explore these systems.
#
# # Game of Life
#
# Game of Life is a popular cellular automaton devised by British mathematician
# John Conway. It consists of a two dimensional grid of cells with a start state
# with each cell being either dead or live at any point in time.
#
# The rules of the system are as follows :-
# * A cell with less than two neighbours dies due to underpopulation.
# * A cell with two or three neighbours lives.
# * A cell with greater than three neighbours dies by overpopulation.
# * A dead cell with three live neighbours becomes live.
#
# We will implement this game of life in minecraft to generate some interesting
# patterns.

using PiCraft

# First let's write our function which will transform from one state to another.

function nextState(A::Array{Bool, 2})
    B = similar(A)
    dims = size(A)
    for i = 1:dims[1], j = 1:dims[2]
        liveNeighbours = 0
        for p = -1:1, q = -1:1
            if checkbounds(Bool, A, i + p, j + q) && (p == 0 && q == 0)
                liveNeighbours += 1
            end
        end
        if A[i, j] == false && liveNeighbours == 3
            B[i, j] = true
        elseif liveNeighbours < 2 || liveNeighbours > 3
            B[i, j] = false
        end
    end
    return B
end

# Now to render an arbitrary 2-D board in Minecraft
# Let live cells be white wool and dead be black wool

function displayState(p::Tuple{Int, Int, Int}, A::Array{Bool, 2})
    dims = size(A)
    for i = 1:dims[1], j = 1:dims[2]
        if A[i, j]
            setBlock(p .+ (i, j, 0), Block(35))
        else
            setBlock(p .+ (i, j, 0), Block(35, 15))
        end
    end
end

p = getTile() .+ (3, 4, 3)
A = rand(Bool, 10, 10)

for i = 1:20
    displayState(A)
    A = nextState(A)
    sleep(0.5)
end

# A variety of interesting patterns can be found ![here](http://web.stanford.edu/~cdebs/GameOfLife/)
# It is encouraged that you experiment with various patterns.

# # 3-D implementation
# If we want to consider a 3-D implementation of the Game of Life then we must
# realise that in 3-D a cell will have 26 Neighbours (3^3 - 1).
#
# We generally express a 3-D system as a `Life wxyz` system.
# The rules of such a system are :
#
# * The number of live neighbours must be between `w` and `x` to sustain life.
# * The number of live neighbours must be between `y` and `z` to bring a dead
#   back to life.
#
# Conway's Game of Life is hence a `Life 2333` system.
# Since a large number of combinations are possible they have been studied
# extensively for their properties. Most systems quickly collapse or explode.
# Life 4555 and 5766 are the only valid systems which exhibit life like
# properties.
#
# Let's implement Life 4555. For that we will rewrite the `nextState` and `displayState`
# funcitons.

function nextState(A::Array{Bool, 3})
    B = similar(A)
    dims = size(A)
    for i = 1:dims[1], j = 1:dims[2], k = 1:dims[3]
        liveNeighbours = 0
        for p = -1:1, q = -1:1, r = -1:1
            if checkbounds(Bool, A, i + p, j + q, k + r) && (p == 0 && q == 0 && r == 0)
                liveNeighbours += 1
            end
        end
        if A[i, j, k] == false && liveNeighbours == 5
            B[i, j, k] = true
        elseif liveNeighbours < 4 || liveNeighbours > 5
            B[i, j, k] = false
        end
    end
    return B
end

function displayState(p::Tuple{Int, Int, Int}, A::Array{Bool, 3})
    dims = size(A)
    for i = 1:dims[1], j = 1:dims[2], k = 1:dims[3]
        if A[i, j, k]
            setBlock(p .+ (i, j, k), Block(35))
        else
            setBlock(p .+ (i, j, k), Block(35, 15))
        end
    end
end

p = getTile() .+ (3, 4, 3)
A = rand(Bool, 7, 7, 7)

for i = 1:20
    displayState(A)
    A = nextState(A)
    sleep(0.5)
end


using Optim, SMM, HTTP, GLM, LinearAlgebra, Random, Statistics, DataFrames, DataFramesMeta, CSV
#Worked with Ahmed and Aleeze
function ps7()
#Q1
url = "https://raw.githubusercontent.com/OU-PhD-Econometrics/fall-2020/master/ProblemSets/PS1-julia-intro/nlsw88.csv"
df = CSV.read(HTTP.get(url).body, DataFrame)
X = [ones(size(df,1),1) df.age df.race.==1 df.collgrad.==1]
y = df.married.==1

function ols_gmm(alpha, X,y)
        g = y .- X*alpha
        J = g'*I*g
        return J
end
alphhat_optim = optimize(a -> ols_gmm(a, X, y), rand(size(X,2)), LBFGS(), Optim.Options(g_tol=1e-8, iterations=100_000))
println(alphhat_optim.minimizer)
# Checking work
bols = inv(X'*X)*X'*y

#2a

df = dropmissing(df, :occupation)
df[df.occupation.==8 ,:occupation] .= 7
df[df.occupation.==9 ,:occupation] .= 7
df[df.occupation.==10,:occupation] .= 7
df[df.occupation.==11,:occupation] .= 7
df[df.occupation.==12,:occupation] .= 7
df[df.occupation.==13,:occupation] .= 7

X = [ones(size(df,1),1) df.age df.race.==1 df.collgrad.==1]
y = df.occupation

function mlogit(alpha, X, y)

    K = size(X,2)
    J = length(unique(y))
    N = length(y)
    bigY = zeros(N,J)
    for j=1:J
        bigY[:,j] = y.==j
    end
    bigAlpha = [reshape(alpha,K,J-1) zeros(K)]

    num = zeros(N,J)
    dem = zeros(N)
    for j=1:J
        num[:,j] = exp.(X*bigAlpha[:,j])
        dem .+= num[:,j]
    end

    P = num./repeat(dem,1,J)

    loglike = -sum( bigY.*log.(P) )

    return loglike
end

alpha_zero = zeros(6*size(X,2))
alpha_rand = rand(6*size(X,2))
alpha_true = [.1910213,-.0335262,.5963968,.4165052,-.1698368,-.0359784,1.30684,-.430997,.6894727,-.0104578,.5231634,-1.492475,-2.26748,-.0053001,1.391402,-.9849661,-1.398468,-.0142969,-.0176531,-1.495123,.2454891,-.0067267,-.5382892,-3.78975]
alpha_start = alpha_true.*rand(size(alpha_true))
println(size(alpha_true))

#2b

alpha_hat_optim = optimize(a -> mlogit(a, X, y), alpha_start, LBFGS(), Optim.Options(g_tol = 1e-5, iterations=100_000, show_trace=true, show_every=50))
alpha_hat_mle = alpha_hat_optim.minimizer
println(alpha_hat_mle)

#2C

function logit_gmm(alpha, X, y)
    K = size(X,2)
    J = length(unique(y))
    N = length(y)
    bigY = zeros(N,J)
    for j=1:J
        bigY[:,j] = y.==j
    end
    bigAlpha = [reshape(alpha,K,J-1) zeros(K)]

    num = zeros(N,J)
    dem = zeros(N)
    for j=1:J
        num[:,j] = exp.(X*bigAlpha[:,j])
        dem .+= num[:,j]
    end

    P = num./repeat(dem,1,J)

    loglike = -sum( bigY.*log.(P) )

    return loglike
end

alpha_hat_optim = optimize(a -> logit_gmm(a, X, y), alpha_true, LBFGS(), Optim.Options(g_tol = 1e-5, iterations=100_000, show_trace=true, show_every=50))
println(alpha_hat_optim)

alpha_hat_optim = optimize(a -> logit_gmm(a, X, y), alpha_rand, LBFGS(), Optim.Options(g_tol = 1e-5, iterations=100_000, show_trace=true, show_every=50))
println(alpha_hat_optim)

#The objective function looks globally concave.

#3a
XR = rand(1000,2)
#3b
beta = [[(.4),(.1),(.5)] [(.5),(.2),(.4)]]]
#3c
p = XR*beta
#3d
eps = random(GeneralizedExtremeValue(0, 1, 0), 1000, 3)

GenY = zeros(1000,1)
for i in 1:size((XR),1)
    GenY[i,1] = argmax(p[i,:] + eps[i,:])
end

#4

MA = SMM.parallelNormal()
dc = SMM.history(MA.chains[1])
dc = dc[dc[:accepted].==true, :]
println(describe(dc))

#5, I'm sorry but I was very confused on this question. I tried mixing and matching with the code you gave in
#the slides but I'm sure I'm missing something here.
function gmm_smm(θ, X, y, D)
    K = size(X,2)
    N = size(y,1)
    β = θ[1:end-1]
    σ = θ[end]
    if length(β)==1
        β = β[1]
    end
    # N+1 moments in both model and data
    gmodel = zeros(N+1,D)
    # data moments are just the y vector itself
    # and the variance of the y vector
    gdata  = vcat(y,var(y))
    #### !!!!!!!!!!!!!!!!!!!!!!!!!!!!! ####
    # This is critical!                   #
    Random.seed!(1234)                    #
    # You must always use the same ε draw #
    # for every guess of θ!               #
    #### !!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ###
    # simulated model moments
    for d=1:D
        ε = σ*randn(N)
        ỹ = X*β .+ ε
        gmodel[1:end-1,d] = ỹ
        gmodel[  end  ,d] = var(ỹ)
    end
    # criterion function
    err = vec(gdata .- mean(gmodel; dims=2))
    # weighting matrix is the identity matrix
    # minimize weighted difference between data and moments
    J = err'*I*err
    return J
end

alpha_hat_smm = optimize(a -> gmm_smm(a, X, y, 1000), alpha_true, LBFGS(), Optim.Options(g_tol = 1e-5, iterations=100_000, show_trace=true, show_every=50))
println(alpha_hat_smm)
end

ps7()


# psf-gauss.jl

export psf_gauss

"""
    psf_gauss( ; ny, px, pz, fwhm_start, fwhm_end, fwhm, fwhm_x, fwhm_z, T)

Create depth-dependent Gaussian PSFs
having specified full-width half-maximum (FHWM) values.

# Options
- 'ny::Int = 128'
- 'px::Int = 11' (should be odd)
- 'pz::Int = px' (should be odd)
- 'fwhm_start::Real = 1'
- 'fwhm_end::Real = 4'
- 'fwhm::AbstractVector{<:Real} = LinRange(fwhm_start, fwhm_end, ny)'
- 'fwhm_x::AbstractVector{<:Real} = fwhm,
- 'fwhm_z::AbstractVector{<:Real} = fwhm_x'
- 'T::DataType == Float32'

Returned `psf` is `[px, pz, ny]` where each PSF sums to 1.
"""
function psf_gauss( ;
    ny::Int = 128,
    px::Int = 11,
    pz::Int = px,
    fwhm_start::Real = 1,
    fwhm_end::Real = 4,
    fwhm::AbstractVector{<:Real} = LinRange(fwhm_start, fwhm_end, ny),
    fwhm_x::AbstractVector{<:Real} = fwhm,
    fwhm_z::AbstractVector{<:Real} = fwhm_x,
    T::DataType = Float32,
)
    isodd(px) || @warn("even px = $px ?")
    isodd(pz) || @warn("even pz = $pz ?")
    psf = zeros(T, px, pz, ny)

    rx = (-(px-1)÷2):((px-1)÷2)
    rz = (-(pz-1)÷2):((pz-1)÷2)
    for iy in 1:ny # depth-dependent blur
        psf[:,:,iy] = psf_gauss(rx, fwhm_x[iy]) * psf_gauss(rz, fwhm_z[iy])'
    end
    return psf
end

function psf_gauss(r::AbstractVector, fwhm::Real)
   if fwhm == 0
        any(==(0), r) || throw("must have some r=0 if fwhm=0")
        return r .== 0 # Kronecker impulse
   end
   σ = fwhm / sqrt(log(256)) # FWHM to Gaussian σ
   psf = @. exp(-0.5 * abs2(r / σ))
   return psf / sum(psf)
end


export problem1
"""
    problem1(N=1000)

# Multiples of 3 and 5
## Problem 1

If we list all the natural numbers below 10 that are multiples of 3 or 5, we get 3, 5, 6 and 9. The sum of these multiples is 23.

Find the sum of all the multiples of 3 or 5 below 1000.
"""
function problem1(N=1000)
    res = 0
    for i in 1:(N-1)
        (mod(i,3) == 0 || mod(i,5) == 0) && (res += i; true)
    end
    return res
end


# "Escaping the swirling vortex"
#
# Nonautonomous ODE from pg 121 of Hale's ODE book, credited to
# Markus, L. and H. Yamabe "Global stability criteria for diferential systems",
# Osaka Math. J. 12 (1960), 305-317.
#
# A nonautonomous linear ODE of the form `du/dt = A(t)u`, with all the eigenvalues
# of `A(t)` being complex with negative real part.
#
# However, origin is an unstable fixed point!
#
# The vector field visualization was based on
# https://gist.github.com/Wikunia/4ca58b4c08da90978c0dcdee5b0c908a

using Javis
using Colors
using LinearAlgebra

"""
    ground(args...)

Define the background and the initial "foreground" color.
"""
function ground(args...)
    background("black")
    sethue("white")
    return nothing
end

"""
dudt(x, y, t)

Return the right hand side of the linear equation `u'=A(t)u`,
where u=[x,y]. In this case, the equation takes the from
    x' = (-1 + 1.5cos(t)^2)x + (1 - 1.5cos(t)sin(t))y
    y' = (-1 - 1.5sin(t)cos(t))x + (-1 + 1.5sin(t)^2)y
"""
function dudt(x, y, t)
    ct = cos(t)
    st = sin(t)
    dxdt = (-1.0 + 1.5 * ct^2) * x + (1.0 - 1.5 * ct * st) * y
    dydt = (-1.0 - 1.5 * st * ct) * x + (-1.0 + 1.5 * st^2) * y
    return dxdt, dydt
end

"""
    jacobian!(mat, t)

Return (and modify in place) the linear part `A(t)` of the equation `u'=A(t)u`
(see [`dudt`](@ref)). This is also seen as the Jacobian of `u -> f(t,u) = A(t)u`.
"""
function jacobian!(mat, t)
    ct = cos(t)
    st = sin(t)
    mat[1, 1] = -1.0 + 1.5 * ct^2
    mat[1, 2] = 1.0 - 1.5 * ct * st
    mat[2, 1] = -1.0 - 1.5 * st * ct
    mat[2, 2] = -1.0 + 1.5 * st^2
    return mat
end

"""
    solution!(u,t)

Return a particular solution of the equation `u'=A(t)u`, given by
`u = exp(t/2)  * [-cos(t), sin(t)] / 20.0` corresponding to the initial condition
`u(0) = [-0.05, 0.0]`
"""
function solution!(u, t)
    et2 = exp(t / 2) / 20.0
    u[1] = -et2 * cos(t)
    u[2] = et2 * sin(t)
    return u
end

"""
    space_to_frame_coordinates(u, frm_scl)

Transform a point in xy coordinates to the frame coordinates XY.
They are related according to `X = frm_scl * x` and `Y = - frm_scl * y`,
for a given scale factor `frm_scl`.
"""
space_to_frame_coordinates(u, frm_scl) = frm_scl * Point(u[1], -u[2])
space_to_frame_coordinates(x, y, frm_scl) = space_to_frame_coordinates([x, y], frm_scl)

"""
    ellipse_level_set!(u, t, mat, σ₁, σ₂, frm_scl)

Draw a "level-set"-like ellipse associated with the solution at time `t`.

For a given time `t`, the solutions `w=w(s)` of the autonomous equation
`dw(s)/ds = A(t)w(s)` spiral towards the origin, decreasing along the level
sets of an elliptic paraboloid. This function computes and draws the ellipse
level set containing the solution `u(t)`. Note, however, that `u=u(t)` is
the solution of the nonautonomous equation `du(t)/dt = A(t)u(t)` given
by [`solution!`](@ref), which spirals away from the origin.

The ellipse is computed by finding the complex eigenvector `v=v(t)` of the
complex eigenvalue `α + iβ` of `A(t)` and considering the real and invertible
matrix `P(t) = [Re(v(t)) Im(v(t))]` which transforms `A(t)` into its real
Jordan form `P(t)^-1 * A(t) P(t) = [α β, -β α]`. In this particular case, the
eigenvalues `α ± iβ` are time-independent. The ellipse is obtained from the
SVD decomposition `P = UΣVᵗ`, where `Σ = diag(σ₁, σ₂)` is diagonal with
nonnegative elements `σ₁ ≥ σ₂ ≥ 0`, and `U` and `V` are orthogonal matrices.
In general, all these terms may change in time, but in the particular
example we are considering, only `U=U(t)` and `V=V(t)` are time-dependent,
with `σ₁ > σ₂ > 0` being constant. The elements `σ₁` and `σ₂` are, precisely,
the semi-major and semi-minor axes of the ellipse associated with the level
set of the elliptic paraboloid at level one. It determines the eccentricity
of the ellipse. The orientation of the ellipse is given by the rotation
determined by the orthogonal matrix `U=U(t)`.

At each time `t`, we find and draw the level set that contains the solution `u(t)`
as follows. We rotate `u(t)` via `Uᵗ = inv(U)`, compute the semi-major
and semi-minor axis of the ellipse that has the same level as `Uᵗu(t)`, compute
the foci of this ellipse, rotate the foci back to the right orientation by applying
the orthogonal matrix `U`, and then find the ellipse, in frame coordinates, that
defined by the foci, in frame coordinates, and passing by the solution `u(t)`, in
frame coordinates.
"""
function ellipse_level_set!(u, t, mat, σ₁, σ₂, frm_scl)
    sethue("white")
    jacobian!(mat, t)
    v = eigen(mat).vectors[:, 1]
    U = svd([real(v) imag(v)]).U
    x, y = transpose(U) * solution!(u, t)
    γ = sqrt(x^2 / σ₁^2 + y^2 / σ₂^2) # level
    σ̃₁ = γ * σ₁ # semi-major
    σ̃₂ = γ * σ₂ # semi-minor
    fd = sqrt(abs(σ̃₁^2 - σ̃₂^2)) # focal distance to the origin/center
    f = U * [fd, 0.0] # one focus point; the other is -f
    poly(
        ellipse(
            space_to_frame_coordinates(f, frm_scl), # focus in frame coordinates
            -space_to_frame_coordinates(f, frm_scl), # the other focus
            space_to_frame_coordinates(u, frm_scl), # ellipse/level containing u(t)
        ),
        :stroke,
        close = true,
    )
    return nothing
end

"""
    ball(p=O, size=4, color="red")

Draw a filled circle of size `size` and color `color` around the point `p`
"""
function ball(p = O, size = 4, color = "red")
    sethue(color)
    circle(p, size, :fill)
    return p
end

"""
    path!(points, pos, color)

Update and draw the trail of the solution.

Add the point `pos` to the vector `points` containing the trail of the trajectory
and draw the trail.
"""
function path!(points, pos, color)
    sethue(color)
    push!(points, pos) # add pos to points
    prettypoly(points) # draw dotted path with circles
    return nothing
end

"""
    ode_arrow(x, y, t, frm_scl, color_min, color_max)

Create one arrow at the base point `(x,y)` and arrow vector given by `dudt(x, y, t)`.
The arrow is properly scaled by the factor `frm_scl` and the orientation takes into
account that the vertical coordinate increases downwards. Depending on the magnitude
of the vector field at that point, the color ranges from `color_min` to `color_max`.
"""
function ode_arrow(x, y, t, frm_scl, color_min, color_max)
    @JShape begin
        translate(space_to_frame_coordinates(x, y, frm_scl))
        dx, dy = dudt(x, y, t)
        delta_t = 0.1
        ξ = dx * delta_t
        η = dy * delta_t
        l = sqrt(ξ^2 + η^2)
        len_colors = 100
        color_range = range(color(color_min), stop = color(color_max), length = len_colors)
        idx = clamp(floor(Int, l * len_colors), 1, len_colors)
        sethue(color_range[idx])
        if ξ != 0 || η != 0
            arrow(
                O,
                space_to_frame_coordinates(ξ, η, frm_scl);
                linewidth = 2,
                arrowheadlength = 8,
            )
        end
    end x = x y = y t = t frm_scl = frm_scl
end

"""
    horizontal_grid_line(video, c, frm_scl)

Create an horizontal grid line at `y=c`
"""
function horizontal_grid_line(video, r, frm_scl)
    if r == 0
        setline(1.5)
    end
    sethue("gray")
    line(Point(-video.width / 2, r * frm_scl), Point(video.width / 2, r * frm_scl), :stroke)
    return nothing
end

"""
    vertical_grid_line(video, c, frm_scl)

Create a vertical grid line at `x=c`.
"""
function vertical_grid_line(video, c, frm_scl)
    if c == 0
        setline(1.5)
    end
    sethue("gray")
    line(
        Point(c * frm_scl, -video.height / 2),
        Point(c * frm_scl, video.height / 2),
        :stroke,
    )
    return nothing
end

"""
    animate()

Create the animation. 😃
"""
function animate()
    # initial setup
    vid = Video(500, 500)
    nframes = 200
    xgrid = -2.0:0.25:2.0 # grid for bars and arrow base points
    ygrid = -2.0:0.25:2.0 # grid for bars and arrow base points
    t₀, tf = 0.0, 5π / 2 # time interval for the simulation
    u = [0.0, 0.0] # dummy initizaliation of the solution vector
    solution!(u, t₀) # initial vector
    mat = zeros(2, 2) # dummy initialization of the jacobian
    jacobian!(mat, t₀) # jacobian at time t₀
    v = eigen(mat).vectors[:, 1] # first eigenvector
    svd_P = svd([real(v) imag(v)]) # P = [real(v) imag(v)] => inv(P) * mat * P = [α β; -β α]
    σ₁, σ₂ = svd_P.S # the semi-major and semi-minor axes are constant in this problem
    t = t₀ # set t to initial time
    frm_scl = 120 # frame scale
    solution_trail = Point[] # initialize history vector for the solution path

    # objects
    ## set background for all frames
    Background(1:nframes, ground)
    ## grids
    [
        Object(1:nframes, (args...) -> horizontal_grid_line(args[1], x, frm_scl)) for
        x in xgrid
    ]
    [Object(1:nframes, (args...) -> vertical_grid_line(args[1], y, frm_scl)) for y in ygrid]
    ## vector field
    arrows = [
        Object(1:nframes, ode_arrow(x, y, t, frm_scl, "yellow", "red")) for x in xgrid,
        y in ygrid
    ]
    ## solution at time t
    sol_pos = Object(
        1:nframes,
        (args...; t) -> ball(
            space_to_frame_coordinates(solution!(u, t), frm_scl),
            6,
            "lightskyblue",
        ),
    )
    ## solution trail
    Object(1:nframes, (args...) -> path!(solution_trail, pos(sol_pos), "lightskyblue"))
    ## ellipse
    obj_ellipse =
        Object(1:nframes, (args...; t) -> ellipse_level_set!(u, t, mat, σ₁, σ₂, frm_scl))

    # animation
    ## update solution, vector field and ellipse
    ## rmk: animation of the solution trail is a byproduct of updating solution
    act!(sol_pos, Action(1:nframes, change(:t, t₀ => tf)))
    act!(arrows, Action(1:nframes, change(:t, t₀ => tf)))
    act!(obj_ellipse, Action(1:nframes, change(:t, t₀ => tf)))

    # rendering
    ## render and generate gif
    render(
        vid;
        framerate = 15,
        pathname = joinpath(@__DIR__, "gifs/escaping_the_swirling_vortex.gif"),
    )

    return nothing
end

animate()


using LinearAlgebra
using Plots

function get_mass2_mat(nu1_mass::Float16, nu2_mass::Float16)::Array
    """Create a 2D mass-squared diagonal matrix"""
    ret_array::Array{Float16} = Diagonal([nu1_mass^2, nu2_mass^2])
    return ret_array
end

function get_mixing_matrix(mixing_angle::Float16, isdeg::Bool = true)::Array
    """Create a 2D mixing/rotation matrix with a given angle"""
    if isdeg
        mixing_angle = deg2rad(mixing_angle)
    end
    mixing_mat::Array{Float16} = [
        +cos(mixing_angle) -sin(mixing_angle)
        +sin(mixing_angle) +cos(mixing_angle)
    ]
    return mixing_mat
end

function time_propagator_mat(
        energy_nu::Float16,
        time_step::Float16,
        delta_mat::Array{Float16}
)::Array{ComplexF16}
    """Calculate the time propagator. Due to a bug in Julia, complex
    matrix arguments for the exponential function are not supported.
    So instead, I use the Euler relation instead.
    """
    exp_arg::Array{ComplexF16} = time_step / (2.0 * energy_nu) * delta_mat
    exp_mat::Array{ComplexF16} = cos(exp_arg) + (0 + 1im) * sin(exp_arg)
    return exp_mat
end

function schrodingerODE()
    """Run a 2D toy example of neutrino oscillations"""

    # Define scalars
    nu1_mass::Float16 = 1.0
    nu2_mass::Float16 = 2.0
    mixing_angle_deg::Float16 = 40
    time_steps::Int = 100
    time_step_size::Float16 = 0.01
    energy_nu::Float16 = 10.0

    # Define physics matrices
    mass2_mat::Array{Float16} = get_mass2_mat(nu1_mass, nu2_mass)
    mixing_mat::Array{Float16} = get_mixing_matrix(mixing_angle_deg, true)
    delta_mat::Array{Float16} = transpose(conj(mixing_mat)) * mass2_mat * mixing_mat

    # Store results in arrays
    # initial state is nu1 = 100%, nu2 = 0%
    initial_state_vector::Vector{ComplexF16} = Vector([1 + 0im, 0 + 0im])
    state_vectors::Array{ComplexF16} = zeros(ComplexF16, (2, time_steps))
    state_fractions::Array{Float16} = zeros(Float16, (2, time_steps))
    state_vectors[:, 1] = initial_state_vector
    state_fractions[:, 1] = Array([1.0; 0.0])

    # Evaluate the wave-function over time and save the state over each step
    for step_index::Int = 2:time_steps
        previous_step_index::Int = step_index - 1
        previous_state_vector::Array{ComplexF16} = state_vectors[:, previous_step_index]
        time_step::Float16 = (step_index - 1) * time_step_size
        time_propagator::Array{ComplexF16} =
            time_propagator_mat(energy_nu, time_step, delta_mat)
        new_state::Vector{ComplexF16} = time_propagator * previous_state_vector
        new_state_fraction::Vector{Float16} =
            Array([abs2(new_state[1]); abs2(new_state[2])])
        state_vectors[:, step_index] = new_state
        state_fractions[:, step_index] = new_state_fraction
    end

    # Plot the results
    Plots.pyplot()
    fig = plot(1:time_steps, state_fractions[1, :], label="\$\\nu_1\$")
    plot!(1:time_steps, state_fractions[2, :], label="\$\\nu_2\$")
    xlabel!("Time step")
    ylabel!("Wave Function Component")
    title!("Flavor Oscillations")
    @show fig
end

schrodingerODE()


"""
Approximates the area under the curve using the trapezoidal rule
Arguments:
 - f: function for the
 - x_start: starting value for x
 - x_end: ending value for x
 - steps: steps taken while integrating.
"""

function trapazoidal_area(f, x_start, x_end, steps)
	x1 = x_start
	fx1 = f(x_start)
	area = 0.0

	for _ in 1:steps
		x2 = ((x_end - x_start) / steps) + x1
		fx2 = f(x2)
		area += abs(fx2 + fx1) * (x2 - x1) / 2
		x1 = x2
		fx1 = fx2
	end

	return area
end


using RDatasets, DataFrames, Lasso, LaTeXStrings, Plots, Measures; pyplot()

df = dataset("MASS", "cpus")
df.Freq = map( x->10^9/x , df.CycT)
df = df[:, [:Perf, :Freq, :MMin, :MMax, :Cach, :ChMin, :ChMax]]
X = [df.Freq df.MMin df.MMax df.Cach df.ChMin df.ChMax]
Y = df.Perf

targetNumVars = 3

lambdaStep = 0.2
lamGrid = collect(0:lambdaStep:150)
lassoFit = fit(LassoPath,X, Y, λ = lamGrid);
dd = Array(lassoFit.coefs)'
nV = sum(dd .!= 0.0 ,dims=2)

goodLambda = lamGrid[findfirst((n)->n==targetNumVars,nV)]
newFit = fit(LassoPath,X, Y, λ = [goodLambda - lambdaStep, goodLambda])
println(newFit)
println("Coefficients: ", Array(newFit.coefs)'[2,:])

p1 = plot(lassoFit.λ, dd, label = ["Freq" "MMin" "MMax" "Cach" "ChMin" "ChMax"],
    ylabel = "Coefficient Value")
plot!([goodLambda,goodLambda],[-1,1.5],c=:black, lw=2, label = "Model cut-off")

p2 = plot(lassoFit.λ,nV, ylabel = "Number of Variables",legend = false)
plot!([goodLambda,goodLambda],[0,6],c=:black, lw=2, label = "Model cut-off")

plot(p1,p2,xlabel= L"\lambda", margin = 5mm, size = (800,400))


using ChaosTools
using Test
using StatsBase

test_value = (val, vmin, vmax) -> @test vmin <= val <= vmax

println("\nTesting generalized entropy (genentropy) & linear scaling...")
@testset "Generalized Dimensions" begin
  @testset "Henon Map" begin
    ds = Systems.henon()
    ts = trajectory(ds, 200000)
    mat = Matrix(ts)
    # Test call with dataset
    genentropy(1, 0.001, ts)
    es = 10 .^ range(-0, stop = -3, length = 7)
    dd = zero(es)
    for q in [0,2,1, 2.56]
      for (i, ee) in enumerate(es)
        dd[i] = genentropy(q, ee, mat)
      end
      linr, dim = linear_region(-log.(es), dd)
      test_value(dim, 1.1, 1.3)
    end
  end
  @testset "Lorenz System" begin
    ds = Systems.lorenz()
    ts = trajectory(ds, 5000)
    es = 10 .^ range(1, stop = -3, length = 11)
    dd = zero(es)
    for q in [0,1,2]
      for (i, ee) in enumerate(es)
        dd[i] = genentropy(q, ee, ts)
      end
      linr, dim = linear_region(-log.(es), dd)
      if q == 0
        test_value(dim, 1.7, 1.9)
      else
        test_value(dim, 1.85, 2.0)
      end
    end
  end
end

println("\nTesting dimension calls (all names)...")
@testset "Dimension calls" begin
  ds = Systems.henon()
  ts = trajectory(ds, 20000)
  # Test call with dataset
  test_value(generalized_dim(1.32, ts), 1.1, 1.3)
  test_value(capacity_dim(ts), 1.1, 1.3)
  test_value(information_dim(ts), 1.1, 1.3)
end

println("\nTesting permutation entropy...")
@testset "Permutation Entropy" begin
    @testset "Trivial entropies" begin
        @test permentropy(zeros(10), 3) == 0
        @test permentropy(ones(10), 3) == 0
        @test permentropy(collect(1:10), 3) == 0
        @test permentropy(collect(10:-1:1), 3) == 0
    end
    @testset "Examples" begin
        # Examples from Bandt & Pompe (2002):
        for (xs, order, desired) in [
                ([4, 7, 9, 10, 6, 11, 3], 2,
                 -(4/6)log2(4/6) - (2/6)log2(2/6)),
                ([4, 7, 9, 10, 6, 11, 3], 3,
                 -2(2/5)log2(2/5) - (1/5)log2(1/5)),
            ]
            @test permentropy(xs, order; base=2) ≈ desired
        end
    end
    @testset "User Interface" begin
        order = Int(typemax(UInt8)) + 1
        @test_throws ArgumentError permentropy([], order)
    end
end


#
#--------------------------------------#
export derivMat
#--------------------------------------#
"""
 Compute derivative matrix for lagrange
 interpolants on points [x]
"""
function derivMat(x)

n = length(x)

a = ones(1,n)
for i=1:n
    for j=1:(i-1) a[i]=a[i]*(x[i]-x[j]) end
    for j=(i+1):n a[i]=a[i]*(x[i]-x[j]) end
end
a = 1 ./ a # Barycentric weights

# diagonal elements
D = x .- x'
for i=1:n D[i,i] = 1. end
D = 1 ./ D
for i=1:n
    D[i,i] = 0.
    D[i,i] = sum(D[i,:])
end

# off-diagonal elements
for j=1:n for i=1:n
    if(i!=j) D[i,j] = a[j] / (a[i]*(x[i]-x[j])) end
end end

return D
end
#--------------------------------------#


using LinearAlgebra

# Parameters in regression model
beta = [148.73, -18.85]; v = 54.40;
y_hat(x) = x'*beta + v;
# Evaluate regression model prediction
x = [0.846, 1]; y = 115;
y_hat(x), y                     # (161.37557999999999, 115)
x = [1.324,2];  y = 234.50;
y_hat(x), y

using VMLS                      # see below to install package
D = house_sales_data();
price = D["price"]
area = D["area"]
beds = D["beds"]

v = 54.4017;
beta = [ 147.7251, -18.8534 ];
predicted = v .+ beta[1] * area + beta[2] * beds;

using Plots
scatter(price, predicted, lims = (0,800));
plot!([0, 800], [0, 800], linestyle = :dash);
# make axes equal and add labels
plot!(xlims = (0,800), ylims = (0,800), size = (500,500));
plot!(xlabel = "Actual price", ylabel = "Predicted price");

# Install VMLS to JuliaBox
# Go to git, enter 'https:////github.com/VMLS-book/VMLS.jl.git'
# Go to package manager, add 'VMLS'
# Wait up to 15 minutes (for package build)

# --OR--

# Install VMLS to Julia runtime (local machine)
# Go to Julia prompt, type ']' to open package manager
# Get library from 'pkg>' prompt
#    type 'add https:////github.com/VMLS-book/VMLS.jl'
#    'ctrl + C' to exit pakage manager prompt


### A Pluto.jl notebook ###
# v0.14.5

using Markdown
using InteractiveUtils

# ╔═╡ 32504772-aaef-11eb-0bb0-0b145c5102db
begin
	using Base.Threads
	using Random
	using Distributions
	using SpecialFunctions
	using Plots
	using StatsPlots
	using Query
	using DataFrames
	using CSV
end

# ╔═╡ ec00bfbc-4889-4fbc-9f5b-ce19b361f1fe
md"""
## Estimating π in Buffon's Needle Problem
First we estimate π many times by simulating Buffon's needle problem. This is repeated for a variety of experimental setups, varying the number of needles thrown (`n_throws`) and the needle length (`needle_length`). For simplicity, we'll just assume that the strip height (`strip_height`) is one - so you can think of the needle length as the ratio of the needle length to the strip height.
"""

# ╔═╡ 7e54380a-fada-40c0-ad20-5795075f2c41
function needle_crossing_fraction(strip_height, needle_length, n_throws)
	D = rand(Uniform(0, strip_height/2), n_throws)
	θ = rand(Uniform(0, π), n_throws)

	crosses_line = (needle_length / 2) * sin.(θ) .> D
	n_crossings = sum(crosses_line)

	return n_crossings / n_throws
end

# ╔═╡ 3de8036e-723b-4805-9384-f87a6aaae5ba
function buffon_pi_estimate(strip_height, needle_length, n_throws)
	p_hat = needle_crossing_fraction(strip_height, needle_length, n_throws)

	return 2 * (needle_length / strip_height) / p_hat
end

# ╔═╡ 9e97dbc2-7d87-40cd-92a8-e892e485d9e6
buffon_pi_estimate(1, 0.1, 10^7)

# ╔═╡ 7838810d-63c0-4de6-8ec7-97a4b65750e2
function estimate_pi_repeatedly(
		n_needle_lengths = 10,
		n_repetitions = 100,
		n_throws_max_exp = 6
	)

	# Set up experiment, arrange data so that (when multi-threading) the
	# processors will be running similar problems at similar times
	df = DataFrame(
		[(needle_length,  n_throws, repetition)
			for n_throws = [10^k for k = 2:n_throws_max_exp]
			for needle_length = [k/n_needle_lengths for k = 1:n_needle_lengths]
			for repetition = 1:n_repetitions]
	)
	rename!(df, ["needle_length", "n_throws", "repetition"])

	# Number of experiments to run
	n_experiments = nrow(df)

	# Run multi-threaded experiments to estimate π
	pi_estimates = zeros(n_experiments)

	Threads.@threads for i ∈ 1:n_experiments
		needle_length = df[i, :needle_length]
		n_throws = df[i, :n_throws]
		pi_estimates[i] = buffon_pi_estimate(1, needle_length, n_throws)
	end

	df.pi_estimate = pi_estimates

	# Evaluate error
	df.error = df.pi_estimate .- π
	df.abs_error = abs.(df.error)

	return df
end

# ╔═╡ bbd5cc5f-80f2-405f-aa71-da0022d74863
pi_estimate_results = estimate_pi_repeatedly(10, 1000, 6)

# ╔═╡ 62e8e764-516a-452a-8f7f-806bf696b126
CSV.write("pi_estimate_results.csv", pi_estimate_results)

# ╔═╡ 27eabc0d-b40f-448f-90ce-301871f3b63b
md"""
## Analyzing the Error in Estimating π
In this section, we take a look at the error committed in estimating π in the various experiments performed above. In particular, we take a look at how the error depends upon the number of needles thrown and how long our needles are. In particular, we take a look at the median error and 95ᵗʰ percentile of our experiments.

In addition, I have computed the theoretical error "bounds" (note this bound is only asymptotically correct / hence more of a guide than a hard and fast "bound"), for 50% and 95% confidence bounds. I didn't bother to plot the actual vs theoretical comparison however, since they're not hard and fast bounds.

I have (however) plotted the observed median and 95ᵗʰ percentile observed error across the experimental set-ups.
"""

# ╔═╡ 6c17402b-8905-463c-aa0f-3a4905877944
md"""

### Analytical Asymptotic Error Bound
Elsewhere, I computed that the following bound holds asymptotically with probability $$P = 1-\alpha$$ (here, $$\Phi^{-1}$$ denotes the probit function, $$h$$ the strip width, $$\ell$$ the needle length, and $$\hat{\pi}$$ the estimate from throwing needles):

$$\vert \hat{\pi} - \pi \vert \lessapprox \pi  \sqrt{\frac{\pi}{2} - \frac{\ell}{h}} \cdot \Phi^{-1}\left(1 - \frac{\alpha}{2}\right)\cdot \sqrt{\frac{h}{\ell}} \cdot \frac{1}{\sqrt{n_\text{throws}}}.$$

We will compare against this below.
"""

# ╔═╡ d75c194b-20df-4ea8-a415-7c44167271d4
function probit(p)
	sqrt(2) * erfinv(2*p - 1)
end

# ╔═╡ 60f54651-6926-4248-aa40-f135d01e4347
function error_bound(needle_length, strip_width, prob_in_interval, n_throws)
	return π * √(π/2 - (needle_length/ strip_width)) *
	                    probit(1 - prob_in_interval/2) *
		                √(strip_width / needle_length) *
	                    1/√(n_throws)
end

# ╔═╡ 3c1ade5a-7a15-4585-8cf2-48b6c84813f7
md"""
The following table shows empirical error bounds, compared against the theoretical approximate bounds.
"""

# ╔═╡ a3304dce-9e87-444d-ab5d-c0f3fffea134
begin
	pi_error_analysis = combine(
		groupby(pi_estimate_results, [:needle_length, :n_throws]),
		:abs_error => median,
		:abs_error => (t -> quantile(t, 0.95))
	)

	rename!(pi_error_analysis, "abs_error_function" => "abs_error_95_pctile")

	transform!(pi_error_analysis, :, [:needle_length, :n_throws] =>
                                     ((a,b) -> error_bound.(a, 1, 0.5, b)) =>
		                             :abs_error_median_theoretical)

	transform!(pi_error_analysis, :, [:needle_length, :n_throws] =>
                                     ((a,b) -> error_bound.(a, 1, 0.05, b)) =>
		                             :abs_error_95_pctile_theoretical)

	pi_error_analysis
end

# ╔═╡ 902e5796-6741-4c4a-9856-a43006c326ab
CSV.write("pi_error_analysis.csv", pi_error_analysis)

# ╔═╡ 7014ffed-1bed-41aa-95ef-ecdcd366b510
md"""
The following plot shows the empirical median absolute error, compared against the theoretical approximate 50% percent confident bound on error.
"""

# ╔═╡ ab93cbb3-6b82-48d7-8fdd-4c91f300d33c
begin
	plot_ref_50th = pi_error_analysis |>
	@df scatter(
		:n_throws,
		:abs_error_median,
		group=:needle_length,
		color=reshape(palette(:auto)[1:10], (1, 10)),
		label=reshape(["" for i =1:10], (1,10)),
		# ugly hack to hide junk entry:
		xaxis=((75, 10^6+5*10^5), :log),
		yaxis=:log
	)

	pi_error_analysis |>
	@df plot!(
		:n_throws,
		:abs_error_median_theoretical,
		group=:needle_length,
		#legend=false
		color=reshape(palette(:auto)[1:10], (1, 10)),
		ylabel="Median of Absolute Error",
		xlabel="Number of Needles Thrown",
		title="Median Absolute Error vs Number of Needles Thrown\nby Needle Length",
		titlefontsize=10,
		legendtitle="Needle Length",
		legendtitlefontsize=8,
		legend = :outertopright

	)

	# junk data to get legend entries
    plot!([], [], label = " ", color = "white")
    plot!([], [], label = "Theoretical", color = "black")
    scatter!([10], [1], label = "Observed", color="black")

	# save figure
	savefig(plot_ref_50th, "error_plot_50pctile.svg")

	# display result in Pluto
	plot_ref_50th
end

# ╔═╡ ed32a816-05d8-46f2-b3fe-6a9c5a2f2636
md"""
The following plot shows the empirical 95ᵗʰ percentile of absolute error, compared against the theoretical 95% confident bound on error.
"""

# ╔═╡ a5f76835-73ce-4fa9-a0b0-e63f1f8bff69
begin
	plot_ref_95th = pi_error_analysis |>
	@df scatter(
		:n_throws,
		:abs_error_95_pctile,
		group=:needle_length,
		color=reshape(palette(:auto)[1:10], (1, 10)),
		label=reshape(["" for i =1:10], (1,10)),
		# ugly hack to hide junk entry:
		xaxis=((75, 10^6+5*10^5), :log),
		yaxis=:log
	)

	pi_error_analysis |>
	@df plot!(
		:n_throws,
		:abs_error_95_pctile_theoretical,
		group=:needle_length,
		#legend=false
		color=reshape(palette(:auto)[1:10], (1, 10)),
		ylabel="95ᵗʰ Percentile of Absolute Error",
		xlabel="Number of Needles Thrown",
		title="95ᵗʰ Percentile of Absolute Error vs Number of Needles Thrown\nby Needle Length",
		titlefontsize=10,
		legendtitle="Needle Length",
		legendtitlefontsize=8,
		legend = :outertopright

	)

	# junk data to get legend entries
    plot!([], [], label = " ", color = "white")
    plot!([], [], label = "Theoretical", color = "black")
    scatter!([10], [1], label = "Observed", color="black")

	# save figure
	savefig(plot_ref_95th, "error_plot_95pctile.svg")

	# display result in Pluto
	plot_ref_95th
end

# ╔═╡ bde8330a-fb98-4950-ad92-c82e2e5e65c6
md"""
### Plotting π Estimates when `needle_length = 1.0`

Since we have a lot of estimates for π, but have only looked at the error we made when estimating them (as opposed to the estimates themselves), I figure I should at least take a look at the estimates in one of the experimental settings.
"""

# ╔═╡ d82dd81c-bf96-443e-8b5a-aa75b21e037e
begin
	df_tmp = filter(row -> row[:needle_length] == 1.0, pi_estimate_results)

	plot_ref_len_1 = df_tmp |>
	@df dotplot(
		string.(:n_throws),
		:pi_estimate,
		ylabel="Estimate for π",
		xlabel="Number of Needles Thrown",
		title="Distribution of π Estimates by Number of Needles Thrown\nFor Needle Length=1.0",
		titlefontsize=11,
		marker=(:lightgrey, stroke(0))
	)

	plot_ref_len_1 = df_tmp |>
	@df boxplot!(
		string.(:n_throws),
		:pi_estimate,
		marker=(:black, stroke(0)),
		fillcolor=:lightblue,
		fillalpha=0.75,
		legend=false
	)

	# save figure
	savefig(plot_ref_len_1, "pi_estimate_boxplot.svg")

	# show figure
	plot_ref_len_1
end

# ╔═╡ b29276d8-e541-4221-b81f-fde2600a8979
md"""
The following table summarizes the results:
"""

# ╔═╡ 312ba44a-dcad-4b18-b130-3439625f78f3
begin
	pi_rslt = combine(
		groupby(pi_estimate_results, [:needle_length, :n_throws]),
		:pi_estimate => mean => :mean,
		:pi_estimate => minimum => :min,
		:pi_estimate => (t -> quantile(t, 0.25)) => :pctile_25,
		:pi_estimate => median => :median,
		:pi_estimate => (t -> quantile(t, 0.75)) => :pctile_75,
		:pi_estimate => maximum => :max,
	)
end

# ╔═╡ b847e348-4afd-4e63-b91b-84152cbb89b7
md"""
Let's take a look at the `needle_length = 1` case.
"""

# ╔═╡ e2042def-bf39-4b95-bdc9-1d46db79c88e
# show an interesting subset
filter(row -> row[:needle_length] == 1.0, pi_rslt)

# ╔═╡ 9b3f7234-9978-43bb-9af9-55bdf967ed62
# save to file
CSV.write("pi_rslt.csv", pi_rslt)

# ╔═╡ 489c446d-f942-4703-b0d7-7a29c98ce4d5
md"""
## How Many Needles Do I Need to Throw?
Finally, I will use the theoretical 95ᵗʰ percentile error bound to estimate how many needles one needs to throw in order to achieve a specified decimal digit accuracy (with 95% confidence) in estimating π.
"""

# ╔═╡ aeb3c67c-5248-4312-9930-852259f43de9
function n_throws_required(desired_accuracy, needle_length, strip_width,
		                   prob_in_interval)

	p = 1 - prob_in_interval

	n_throws =  π^2 * (π/2 - (needle_length / strip_width)) *
	                  probit(1 - p/2)^2 *
		              (strip_width / needle_length) *
	                  (1 / desired_accuracy^2)

	return n_throws
end

# ╔═╡ b988a3e4-1d15-4abf-8b5a-fc8a2e119c7e
begin
	df_required_analytical = DataFrame([
	 (decimal_digits, needle_length, n_throws_required(1/10^decimal_digits,                                                                  needle_length, 1, 0.95))
			for decimal_digits = [1, 2, 3, 4, 5, 6]
			for needle_length = [k/10 for k = 1:10]
	])

	rename!(df_required_analytical, ["decimal_digits", "needle_length",                                                    "throws_required"])
end

# ╔═╡ 55234b47-1b22-4fab-a449-c43b2976b7fa
md"""
Let's take a look at the `needle_length = 1` case.
"""

# ╔═╡ 9668af8a-ecea-4c11-9c3a-f32b416f02a9
# show an interesting subset
filter(row -> row[:needle_length] == 1.0, df_required_analytical)

# ╔═╡ fd88ca22-a191-4e92-900f-c86c7f516641
CSV.write(
	"required_throws_analytical_solution.csv",
	df_required_analytical
)

# ╔═╡ 70b92add-4a58-4181-8409-c896b57665f9
begin
	plot_ref_throws_req = df_required_analytical |>
	@df plot(
		:decimal_digits,
		:throws_required,
		group=:needle_length,
		yaxis=:log,
		ylabel="Number of Throws Required",
		xlabel="Number of Decimal Digits",
		title="Number of Throws Required to Achieve Specified\nDecimal Digit Accuracy with 95% Confidence\nby Needle Length",
		titlefontsize=11,
		legendtitle="Needle Length",
		legendtitlefontsize=8,
		legend = :outertopright
	)

	# save figure
	savefig(plot_ref_throws_req, "throws_required_95pctile.svg")

	# display plot
	plot_ref_throws_req
end

# ╔═╡ Cell order:
# ╟─ec00bfbc-4889-4fbc-9f5b-ce19b361f1fe
# ╠═7e54380a-fada-40c0-ad20-5795075f2c41
# ╠═3de8036e-723b-4805-9384-f87a6aaae5ba
# ╠═9e97dbc2-7d87-40cd-92a8-e892e485d9e6
# ╠═7838810d-63c0-4de6-8ec7-97a4b65750e2
# ╠═bbd5cc5f-80f2-405f-aa71-da0022d74863
# ╠═62e8e764-516a-452a-8f7f-806bf696b126
# ╟─27eabc0d-b40f-448f-90ce-301871f3b63b
# ╟─6c17402b-8905-463c-aa0f-3a4905877944
# ╠═d75c194b-20df-4ea8-a415-7c44167271d4
# ╠═60f54651-6926-4248-aa40-f135d01e4347
# ╟─3c1ade5a-7a15-4585-8cf2-48b6c84813f7
# ╠═a3304dce-9e87-444d-ab5d-c0f3fffea134
# ╠═902e5796-6741-4c4a-9856-a43006c326ab
# ╟─7014ffed-1bed-41aa-95ef-ecdcd366b510
# ╠═ab93cbb3-6b82-48d7-8fdd-4c91f300d33c
# ╟─ed32a816-05d8-46f2-b3fe-6a9c5a2f2636
# ╠═a5f76835-73ce-4fa9-a0b0-e63f1f8bff69
# ╟─bde8330a-fb98-4950-ad92-c82e2e5e65c6
# ╠═d82dd81c-bf96-443e-8b5a-aa75b21e037e
# ╟─b29276d8-e541-4221-b81f-fde2600a8979
# ╠═312ba44a-dcad-4b18-b130-3439625f78f3
# ╟─b847e348-4afd-4e63-b91b-84152cbb89b7
# ╠═e2042def-bf39-4b95-bdc9-1d46db79c88e
# ╠═9b3f7234-9978-43bb-9af9-55bdf967ed62
# ╟─489c446d-f942-4703-b0d7-7a29c98ce4d5
# ╠═aeb3c67c-5248-4312-9930-852259f43de9
# ╠═b988a3e4-1d15-4abf-8b5a-fc8a2e119c7e
# ╟─55234b47-1b22-4fab-a449-c43b2976b7fa
# ╠═9668af8a-ecea-4c11-9c3a-f32b416f02a9
# ╠═fd88ca22-a191-4e92-900f-c86c7f516641
# ╠═70b92add-4a58-4181-8409-c896b57665f9
# ╠═32504772-aaef-11eb-0bb0-0b145c5102db


multsum(n, lim) = (occ = div(lim-1, n); div(n*occ*(occ+1), 2))
multsum(n, m, lim) = multsum(n, lim) + multsum(m, lim) - multsum(lcm(n,m), lim)


using LinearAlgebra
using PyPlot
function linesegment_intersect(p1, p2, q1, q2, δ)
    A = [p2-p1 q1-q2]
    b = q1-p1
    if rank(A) == 2
        st = A \ b
        if all(-δ .≤ st .≤ 1 + δ)
            pintersect = p1 + st[1] * (p2 - p1)
            return true, pintersect
        else
            return false, nothing
        end
    else
        # Warning: parallel lines, should check if they actually overlap
        return false, nothing
    end
end

# Example: n random lines, find and plot all intersections
n = 10
lines = [ [rand(2), rand(2)] for i = 1:n ]

# Plot all lines
for line in lines
    plot([line[1][1], line[2][1]], [line[1][2], line[2][2]])
end

# Find all intersections - note that we only consider j>i to avoid checking
# the same 2 line-segments twice
for i = 1:n
    for j = i+1:n
        isect, pq = linesegment_intersect(lines[i][1], lines[i][2], lines[j][1], lines[j][2], 0)
        if isect
            plot(pq[1], pq[2], "o")
        end
    end
end

axis("equal"); grid(true);


export ladder_graph

"""
    ladder_adj(w1, w2)
return true if words `w1` and `w2` have the same length
and differ in exactly one location.
"""
function ladder_adj(w1::String, w2::String)::Bool
    if length(w1) != length(w2)
        return false
    end
    c = collect(w1) .!= collect(w2)
    return sum(c) == 1
end

"""
    ladder_graph(S::Set{String}, len::Int=0, trim::Bool=false)

Create a graph whose vertices are words in which two words are adjacent if they differ
in exactly one letter.

We use only words of length `len`, unless this is set to `0`
in which case all words are used.

If `trim` is set to `true` then vertices of degree zero are deleted.
"""
function ladder_graph(S::Set{String}, len::Int = 0, trim::Bool = false)::SimpleGraph{String}
    G = _bare_graph(S, len)

    PM = Progress(NV(G))

    for v ∈ G.V
        next!(PM)
        nv = length(v)
        for i ∈ 1:nv
            for c ∈ _A2Z
                w = v[1:i-1] * c * v[i+1:nv]
                if w ∈ G.V
                    add!(G, v, w)
                end
            end
        end
    end

    # remove vertices of degree zero (if trim is true)
    if trim
        for w in G.V
            if deg(G, w) == 0
                delete!(G, w)
            end
        end
    end

    if len < 1
        name(G, "Ladder Graph on all words")
    else
        name(G, "Ladder Graph on $len-letter words")
    end

    return G

end


"""
    luhn(str)

Return true if `str` encodes a valid number in the luhn formula.

A valid string will:

 - Contain only 0-9 and space
 - Contain at least two numbers
 - After a transformation, the sum of the transformed digits will be evenly divisible by 10

The transformation is:

 - Every second digit, counting the rightmost digit as first, is doubled
 - If a doubled digit is greater than 9, subtract 9 from it.
"""
function luhn(str)
    acc = 0
    len = 0
    for chr in Iterators.reverse(str)
        if isdigit(chr)
            len += 1
            x = parse(Int, chr)
            acc += iseven(len) ? (2x < 10 ? 2x : 2x-9) : x
        elseif chr != ' '
            return false
        end
    end
    len > 1 ? acc % 10 == 0 : false
end


solve_liner = Symbolics.solve_for

function solve(problem::LinearProblem)
    eqs = problem.eqs
    vars = problem.vars
    sol = solve_liner(eqs, vars)
    return Dict([(vars[i], sol[i]) for i in 1:length(vars)])
end

function one_cal(array, dict, f, ja)
    return array .- inv(Symbolics.value.(substitute.(ja, (dict,)))) * Symbolics.value.(substitute.(f, (dict,)))
end

function get_dict(array, dict, vars)
    for i in 1:length(vars)
        dict[vars[i]] = array[i]
    end
    return dict
end

function _solve(dict, f, ja, maxiter, abstol, vars)
    last_v = Inf
    new_v = collect(values(dict))
    dict = dict
    count = 0

    while count < maxiter && maximum(abs.(new_v .- last_v)) > abstol
        last_v = new_v
        new_v = one_cal(new_v, dict, f, ja)
        dict = get_dict(new_v, dict, vars)
        count += 1
    end

    if count >= maxiter
        error("Error: iterate times exceed maxiter")
    end

    return dict
end

function solve(problem::NonlinearProblem)
    eqs = problem.eqs
    dict_u = problem.vars
    vars = collect(keys(problem.vars))
    maxiters = problem.maxiters
    abstol = problem.abstol
    jac = Symbolics.jacobian(eqs, vars)
    sol = _solve(dict_u, eqs, jac, maxiters, abstol,vars)
    return sol
end


"""
    gauss_lobatto_quad(α, β, N)

Computes Legendre-Gauss-Lobatto quadrature points and weights with Jacobi weights α,β.
"""
function gauss_lobatto_quad(α::Int, β::Int, N)

    if !(iszero(α) && iszero(β))
        error("alpha/beta must be zero for Lobatto")
    end

    x = zeros(N+1)
    w = zeros(N+1)
    if N == 0
        x[1] = 0.
        w[1] = 2.
    elseif N == 1
        x[1] = -1.0
        x[2] = 1.0
        w[1] = 1.0
        w[2] = 1.0
    elseif N==2
        x[1] = -1.
        x[2] = 0.
        x[3] = 1.
        w[1] = 0.333333333333333
        w[2] = 1.333333333333333
        w[3] = 0.333333333333333
    else
        xint, wint = gauss_quad(α+1, β+1, N-2)
        x = vcat(-1,xint,1)
        wend = 2/(N*(N+1))
        w = vcat(wend, (@. wint / (1-xint^2)) ,wend)
    end
    return x[:],w[:]
end

"""
    gauss_quad(α, β, N)

Compute nodes and weights for Gaussian quadrature with Jacobi weights (α,β).
"""
function gauss_quad(α, β, N)
    if N == 0
        x = [-(α-β)/(α+β+2)]
        w = [2.]
        return x, w
    end

    J = zeros(N+1, N+1)
    h₁ = @. 2*(0:N)+α+β
    J = diagm(0 => @. -1/2*(α^2-β^2)/(h₁+2)/h₁) +
        diagm(1 => @. 2/(h₁[1:N]+2) * sqrt((1:N) * ((1:N)+α+β) * ((1:N)+α) * ((1:N)+β) / (h₁[1:N]+1) / (h₁[1:N]+3)))
    if α + β < 10*eps()
        J[1,1] = 0.0
    end

    x, V = eigen(Symmetric(J+J'))
    w = @. transpose(V[1,:])^2 * 2^(α+β+1) / (α+β+1) * gamma(α+1) * gamma(β+1) / gamma(α+β+1)
    return x[:], w[:]
end

"""
    jacobiP(x, α, β, N, tmp_array=zeros(N+1))
    jacobiP(x::AbstractArray, α, β, N)

Evaluate the Jacobi Polynomial (α, β) of order N at points x.
Can optionally specify `tmp_array` of length > N+1 to avoid allocations.
"""
function jacobiP(x, α, β, N, tmp_array=zeros(N+1))

    PL = tmp_array
    γ₀ = 2^(α+β+1) / (α+β+1) * gamma(α+1) * gamma(β+1) / gamma(α+β+1)
    PL[1] = 1.0 / sqrt(γ₀)
    if N == 0
        return PL[1]
    end

    γ₁ = (α+1) * (β+1) / (α+β+3) * γ₀
    PL[2] = ((α+β+2) * x/2 + (α-β)/2) / sqrt(γ₁)
    if N == 1
        return PL[N+1]
    end

    aold = 2 / (2+α+β) * sqrt((α+1) * (β+1) / (α + β + 3))

    for i = 1:N-1
        h₁ = 2i + α + β
        anew = 2 / (h₁+2) * sqrt((i+1) * (i+1+α+β) * (i+1+α) * (i+1+β) / (h₁+1) / (h₁+3))
        bnew = -(α^2-β^2) / h₁ / (h₁+2)
        PL[i+2] = 1 / anew * (-aold * PL[i] + (x - bnew) * PL[i+1])
        aold = anew
    end

    return PL[N+1]
end

# vectorized version.
# TODO: deprecate?
function jacobiP(x::AbstractArray, α, β, N)

    PL = zeros(N+1, length(x))
    γ₀ = 2^(α+β+1) / (α+β+1) * gamma(α+1) * gamma(β+1) / gamma(α+β+1)
    @. PL[1,:] = 1.0/sqrt(γ₀)
    if N == 0
        return vec(PL)
    end

    γ₁ = (α+1) * (β+1) / (α+β+3) * γ₀
    @. PL[2, :] = ((α+β+2) * x/2 + (α-β)/2) / sqrt(γ₁)
    if N == 1
        return PL[N+1,:]
    end

    aold = 2 / (2+α+β) * sqrt((α+1) * (β+1) / (α + β + 3))

    for i = 1:N-1
        h₁ = 2i+α+β
        anew = 2 / (h₁+2) * sqrt((i+1) * (i+1+α+β) * (i+1+α) * (i+1+β) / (h₁+1) / (h₁+3))
        bnew = -(α^2-β^2) / h₁ / (h₁+2)
        @. PL[i+2,:] = 1 / anew * (-aold * PL[i,:] .+ (x .- bnew) .* PL[i+1,:])
        aold = anew
    end

    return PL[N+1,:]
end

"""
    grad_jacobiP(r, α, β, N, tmp_array=())

Evaluate derivative of Jacobi polynomial (α, β) of order N at points r.
Pass in `tmp_array = tuple(zeros(N+1))` to avoid allocations.
"""

function grad_jacobiP(r, α, β, N, tmp_array=())
    if N != 0
        return sqrt(N * (N+α+β+1)) * jacobiP(r, α+1, β+1, N-1, tmp_array...)
    else
        return zero(r)
    end
end


using OceanTurb, Printf

@use_pyplot_utils

# # Operations on fields
#
# Generate a grid with 100 points and height H=10. By default the
# base of the grid is at z=-H and the top of the grid is at z=0.
grid = UniformGrid(100, 10)

# Define a function that will generate our field's data.
c0(z) = exp(1.2z)

# Create a field of zeros on the grid.
c = CellField(grid)

# Set c to the function c0.
OceanTurb.set!(c, c0)

# Calculate the derivative of c and set it to cz.
cz = ∂z(c)

# Plot the results
plot(c, label=L"c = e^{1.2z}")
plot(cz, label=L"\partial_z c = 1.2 e^{1.2z}")
legend()

title("A field and it's derivative")
xlabel("\$ c \$ and \$ \\partial_z c \$")
ylabel(L"z")

# # Interpolation
#
# Next we demonstrate how to interpolate from one grid to another,
# respecting the integral budget of a quantity.

grid1 = UniformGrid(30, 10)
grid2 = UniformGrid(10, 10)
grid3 = UniformGrid(3, 10)

c1 = CellField(grid1)
c2 = CellField(grid2)
c3 = CellField(grid3)

OceanTurb.set!(c1, c)
OceanTurb.set!(c2, c)
OceanTurb.set!(c3, c)

@show integral(c)
@show integral(c1)
@show integral(c2)
@show integral(c3)

plot(c, ".", label="high-res \$ c \$")
plot(c1, "^", label="medium-res \$ c \$")
plot(c2, "x", label="low-res \$ c \$")
plot(c3, "+", label="crazy low-res \$ c \$")

legend()

title("\$ e^{1.2z} \$ at different resolutions")
xlabel(L"c")
ylabel(L"z")


N = Int(5e1)  # number of data
M = Int(1e5)  # number of realisations

mu = [7; 10] * 1.0  # mean of data
c = [5 3; 3 7] * 1.0  # covarianza en float

# generate data
using Distributions
probMVgaus = MvNormal(mu,c)
x = rand(probMVgaus, N*M)
x = reshape(x, N, M, 2)

# estimo los mu y c de cada relizacion
muest = mean(x, 1)
dif = reshape(x .- muest,N,M,2,1) # ver como hacer esta resta facilmente
cest = sum(dif .* permutedims(dif,[1,2,4,3]), 1) / (N - 1)

# saco la media y varianza de los estimadores entre todas las realizaciones
# media y var de mu
muExp = mean(muest, 2)
difmu = reshape(muest .- muExp, M, 2, 1)
muVar = sum(difmu .* permutedims(difmu,[1,3,2]), 1) / (M - 1)
# media y var de c
cExp = mean(cest, 2)
difc = reshape(cest .- cExp, M,2,2,1,1)
cVar = sum(difc .* permutedims(difc, [1,5,4,3,2]), 1) / (M - 1)
cVar = cVar[1,:,:,:,:]

# saco las cuentas analiticas
expMu = mu
VarMu = c / N
expC = c
# no es necesario trasponer porque c es simetrica
cShaped = reshape(c,2,2,1,1)
VarCnn = cShaped .* permutedims(cShaped, [4,3,2,1])
nCons = (2 * N - 1) / (N - 1)^2
VarC = VarCnn * nCons

print("numerico: \n", cVar, "\n analitico \n", VarC)
print(cVar / nCons)
print(VarCnn)


discount(t, κ) = 1/(1 + κ*t)

function prob(κ, τ, t_ss, t_ll, r_ss, r_ll)
    u_ll = r_ll * discount(t_ll, κ)
    u_ss = r_ss * discount(t_ss, κ)
    return 1/(1 + exp(-τ * (u_ll - u_ss)))
end

function loglike(κ, τ, t_ss, t_ll, r_ss, r_ll, data)
    p = prob(κ, τ, t_ss, t_ll, r_ss, r_ll)
    p = p == 1 ? 1 - eps() : p
    p = p == 0 ? eps() : p
    LL = data ? log(p) : log(1 - p)
    # println(" choice ", data, " kappa ", κ, " tau ", τ, " t_ss ", t_ss,
    #     " t_ll ", t_ll, " r_ss ", r_ss, " r_ll ", r_ll, " LL ", LL)
    return LL
end

function simulate(κ, τ, t_ss, t_ll, r_ss, r_ll)
    p = prob(κ, τ, t_ss, t_ll, r_ss, r_ll)
    return rand() ≤ p ? true : false
end


function testDCT1d(N=32)
  Random.seed!(1235)
  x = zeros(ComplexF64, N^2)
  for i=1:5
    x .+= rand()*cos.(rand(1:N^2)*collect(1:N^2)) .+ 1im*rand()*cos.(rand(1:N^2)*collect(1:N^2))
  end
  D1 = DCTOp(Float64,(N^2,),2)
  D2 = sqrt(2/N^2)*[cos(pi/(N^2)*j*(k+0.5)) for j=0:N^2-1,k=0:N^2-1]
  D2[1,:] .*= 1/sqrt(2)
  D3 = DCTOp(Float64,(N^2,),4)
  D4 = sqrt(2/N^2)*[cos(pi/(N^2)*(j+0.5)*(k+0.5)) for j=0:N^2-1,k=0:N^2-1]

  y1 = D1*x
  y2 = D2*x

  @test norm(y1 - y2) / norm(y1) ≈ 0 atol=0.01

  y3 = D3*x
  y4 = D4*x
  @test norm(y3 - y4) / norm(y1) ≈ 0 atol=0.01

  x1 = adjoint(D1)*y1
  @test norm(x1 - x) / norm(x) ≈ 0 atol=0.01

  x2 = adjoint(D3)*y3
  @test norm(x2 - x) / norm(x) ≈ 0 atol=0.01
end

function testFFT1d(N=32,shift=true)
  Random.seed!(1234)
  x = zeros(N^2)
  for i=1:5
    x .+= rand()*cos.(rand(1:N^2)*collect(1:N^2))
  end
  D1 = FFTOp(Float64,(N^2,),shift)
  D2 =  1.0/N*[exp(-2*pi*im*j*k/N^2) for j=0:N^2-1,k=0:N^2-1]

  y1 = D1*x
  if shift
    y2 = fftshift(D2*fftshift(x))
  else
    y2 = D2*x
  end
  @test norm(y1 - y2) / norm(y1) ≈ 0 atol=0.01

  x1 = adjoint(D1) * y1
  if shift
    x2 = ifftshift(adjoint(D2)*ifftshift(y2))
  else
    x2 = adjoint(D2)*y2
  end
  @test norm(x1 - x2) / norm(x1) ≈ 0 atol=0.01
end

function testFFT2d(N=32,shift=true)
  Random.seed!(1234)
  x = zeros(N^2)
  for i=1:5
    x .+= rand()*cos.(rand(1:N^2)*collect(1:N^2))
  end
  D1 = FFTOp(Float64,(N,N),shift)

  idx = CartesianIndices((N,N))[collect(1:N^2)]
  D2 =  1.0/N*[ exp(-2*pi*im*((idx[j][1]-1)*(idx[k][1]-1)+(idx[j][2]-1)*(idx[k][2]-1))/N) for j=1:N^2, k=1:N^2 ]

  y1 = D1*x
  if shift
    y2 = D2*vec(fftshift(reshape(x,N,N)))
    y2 = vec(fftshift(reshape(y2,N,N)))
  else
    y2 = D2*x
  end
  @test norm(y1 - y2) / norm(y1) ≈ 0 atol=0.01

  x1 = adjoint(D1) * y1
  if shift
    x2 = adjoint(D2)*vec(ifftshift(reshape(y2,N,N)))
    x2 = vec(ifftshift(reshape(x2,N,N)))
  else
    x2 = adjoint(D2)*y2
  end
  @test norm(x1 - x2) / norm(x1) ≈ 0 atol=0.01
end

function testWeighting(N=512)
  Random.seed!(1234)
  x1 = rand(N)
  weights = rand(N)
  W = WeightingOp(weights)
  y1 = W*x1
  y = weights .* x1

  @test norm(y1 - y) / norm(y) ≈ 0 atol=0.01

  x2 = rand(2*N)
  W2 = WeightingOp(weights,2)
  y2 = W2*x2
  y = repeat(weights,2) .* x2

  @test norm(y2 - y) / norm(y) ≈ 0 atol=0.01
end

@testset "Linear Operators" begin
  @info "test DCT-II and DCT-IV"
  for N in [2,8,16,32]
    testDCT1d(N)
  end
  @info "test FFT"
  for N in [8,16,32]
    testFFT1d(N,false)
    testFFT1d(N,true)
    testFFT2d(N,false)
    testFFT2d(N,true)
  end
  @info "test Weighting"
  testWeighting(512)
end


using Plots

function booth(x)
    return (x[1] + 2x[2] - 7)^2 + (2x[1] + x[2] - 5)^2;
end

# plot

contour(-30:0.1:30, -30:0.1:30, (x, y)->booth([x, y]))


# This file investigates mean square error (MSE) with respect to number of interpolation points in 1D interpolation

using FractalTools
using Makie

# Construct interpolation data
f(x) = sin(2π * x)
xi = 0.
xf = 1.
freevar = 0.001
npts = 10 : 2 : 50
ntpts = 10 * npts[end]

# Compute errors
mse = map(npts) do npt
    # Construct interpolation data
    x = collect(range(xi, xf, length=npt))
    y = f.(x)
    pts = collect.(zip(x, y))

    # Construct interpolation
    interp = interpolate(pts, Interp1D(freevar))

    # Construct test data
    xt = collect(range(xi, xf, length=ntpts))

    # Compute error
    sum((f.(xt) - interp.(xt) ).^2) / length(xt)
end

# Plot mse
fig = Figure()
ax = fig[1, 1] = Axis(fig, xlabel="Number of Points", ylabel="MSE", title="1D Interpolation MSE")
stem!(ax, npts, mse, color=:black)
save(joinpath(@__DIR__, "interp1d_mse.png"), fig)
display(fig)


set_theory = [
    Axiom("Definition of the universal set", P ⊂ Ω, TRUE),
    Axiom("Definition of the universal set", x ∈ Ω, TRUE),
    Axiom("Definition of the empty set", ∅ ⊂ A, TRUE),
    Axiom("Definition of the empty set", x ∈ ∅, FALSE),
    Axiom("Equal sets are subsets", A ⊂ A, TRUE),

    Axiom("Definition of Element Of", x ∈ (A ∩ B), (x ∈ A) ∧ (x ∈ B)),
    Axiom("Definition of Element Of", (x ∈ A) ∧ (x ∈ B), x ∈ (A ∩ B)),
    Axiom("Definition of Element Of", x ∈ (A ∪ B), (x ∈ A) ∨ (x ∈ B)),
    Axiom("Definition of Element Of", (x ∈ A) ∨ (x ∈ B), x ∈ (A ∪ B)),

    Axiom("Flatten Condition Set", ConditionSet(y, y ∈ A), A),  # todo: change to f(x)
    Axiom("Expand Condition Set", A, ConditionSet(_y, _y ∈ A)),  # todo: change to f(x)
    Axiom("Definition of Subset", A ⊂ B, (_y ∈ A) ⟹ (_y ∈ B)),
    Axiom("Definition of Subset", (_y ∈ A) ⟹ (_y ∈ B), A ⊂ B),

    Axiom("Definition of the universal set", x ∈ (P ∪ Ω), x ∈ Ω),

    #Theorem("Intersections are subsets of Unions", x ∈ (A ∩ B), x ∈ (A ∪ B), Proof([])),
    #Theorem("Intersections are subsets of Unions", (A ∩ B) ⊂ (A ∪ B), TRUE, Proof([])),
]


#  Negative Binomial type 1 and type II log likelihood
#  The parameterization follows
#  Deb and Trivedi, J. Appl. Econometrics,
#  V. 12, 1997, pp. 313 - 336.
using SpecialFunctions, Distributions
function  negbin(θ, y, x, nbtype)
    n, k = size(x)
    β = θ[1:k]
    eps = 1e-8
    λ = eps .+ exp.(x*β)
    α = eps .+ exp(θ[end])
    nbtype == 1 ? r = λ./α : r = ones(n)/α
    p = r ./ (r + λ)
    all(r .> 0.0) & all(p .> 0.0) & all(p .< 1.0) ? log.(pdf.(NegativeBinomial.(r, p),y)) : -Inf
end


using LinearAlgebra

# Define and use function
f(x) = x[1] + x[2] - x[4]^2     # f(x) = x(1) + x(2) − x(4)^2
f([-1,0,1,2])                   # -5

# Superposition
# eval f at a linear combination of 2 vectors
# == forming the linear combination of f eval at the 2 vectors

a = [-2, 0, 1, -3];
f(x) = a'*x

x=[2,2,-1,1]; y=[0,1,-1,0];
alpha = 1.5;  beta = -3.7;
lhs = f(alpha * x + beta * y)

rhs = alpha * f(x) + beta * f(y)

# Verify superposition for f(e(3)) = a(3)
e3 = [0, 0, 1, 0];
f(e3)                           # 1.0

using Statistics
avg(x) = (ones(length(x)) / length(x))'*x;
x = [1, -3, 2, -1];
avg(x)


function conv(signal1::Vector{Complex}, signal2::Vector{Complex})
    n = length(signal1) + length(signal2)
    out = Vector{Complex}(n)
    sum = 0

    for i = 0:n
        for j = 0:i
            if(j < length(signal1))
                sum += signal1[j] * signal2[i-j]
            end
        end
        out[i] = sum
        sum = 0
    end

    return out
end

function conv_fft(signal1::Vector{Complex}, signal2::Vector{Complex})
    return ifft(fft(signal1).*fft(signal2))
end


using Test
import Roots.newton, Roots.halley

@testset "Test Newton, Halley, Schroder methods" begin

    @test abs(newton(sin, cos, 0.5) - 0.0) <= 100*eps(1.0)
    @test newton(cos, x -> -sin(x), 1.0)  ≈ pi/2
    @test newton(x -> x^2 - 2x - 1, x -> 2x - 2, 3.0)  ≈ 2.414213562373095
    @test abs(newton(x -> exp(x) - cos(x), x -> exp(x) + sin(x), 3.0) - 0.0) <= 1e-14
    @test halley(x -> x^2 - 2x - 1,x -> 2x - 2,x -> 2, 3.0)  ≈ 2.414213562373095
    a = halley(x -> exp(x) - cos(x),
               x -> exp(x) + sin(x),
               x -> exp(x) + cos(x), 3.0)
    @test abs(a - 0.0) <= 1e-14

    # find_zero calls
    @test find_zero((x -> x^2 - 2x - 1,x -> 2x - 2), 3.0, Roots.Newton())  ≈ 2.414213562373095
    @test find_zero((x -> x^2 - 2x - 1,x -> 2x - 2,x -> 2), 3.0, Roots.Halley())  ≈ 2.414213562373095
    @test find_zero((x -> x^2 - 2x - 1,x -> 2x - 2,x -> 2), 3.0, Roots.Schroder())  ≈ 2.414213562373095

    ## test with Complex input

    @test real(Roots.newton(x ->  x^3 - 1, x ->  3x^2, 1+im)) ≈ 1.0
    @test real(Roots.newton(x ->  x^3 - 1, x ->  3x^2, 1+10im)) ≈ (-1/2)

    ## Issue #143 test with new interface
    Roots.newton(sin, cos, 3.0) ≈ π # uses find_zero
    Roots.newton((sin,cos), 3.0) ≈ π # uses simple

    fdf = x -> (sin(x), sin(x)/cos(x))  # (f, f/f')
    @test Roots.find_zero(fdf, 3.0, Roots.Newton())  ≈ π # uses find_zero
    Roots.newton(fdf, 3.0)  ≈ π # uses simple

    fdfdf = x -> (sin(x), sin(x)/cos(x), -cos(x)/sin(x), sin(x)/cos(x))   # (f, f/f', f'/f'', f''/f''')
    @test Roots.find_zero(fdfdf, 3.0, Roots.Halley()) ≈ π

    # check that functions with multiple return values can work with other
    # methods
    for M in [Roots.Schroder(), Roots.Halley(), Roots.Newton(), Roots.Order1(), Roots.Order0()]
        @test Roots.find_zero(fdfdf, 3.0, M) ≈ π  # can pass function to others
    end

    for M in [Roots.Bisection(), Roots.A42(), Roots.AlefeldPotraShi()]
        @test Roots.find_zero(fdfdf, (3.0, 4.0), M) ≈ π  # can pass function to others
    end

    @test find_zero(x -> (x^2 -2, (x^2-2)/2x), 1.0, Roots.Newton(), Roots.Bisection()) ≈ sqrt(2)

    @test_throws Roots.ConvergenceFailed Roots.newton((x->x^2 + 1, x->2x), 0)

end

@testset "Lith Boonkkamp IJzerman methods" begin

    x₀, x̃₀, α = 1.0, 1.1, 1.1673039782614187
    f(x) = x^5 - x - 1
    fp(x) = 5x^4 - 1
    fpp(x) = 20x^3
    fppp(x) = 60x
    fpppp(x) = 60

    @test solve(ZeroProblem((f,), x₀), Roots.LithBoonkkampIJzerman(3,0)) ≈ α
    @test solve(ZeroProblem(f, x₀), Roots.LithBoonkkampIJzerman(3,0)) ≈ α
    @test solve(ZeroProblem((f,), x₀), Roots.LithBoonkkampIJzerman(4,0)) ≈ α
    @test solve(ZeroProblem(f, x₀), Roots.LithBoonkkampIJzerman(4,0)) ≈ α

    @test solve(ZeroProblem((f,fp), x₀), Roots.LithBoonkkampIJzerman(2,1)) ≈ α
    @test solve(ZeroProblem((f,fp), x₀), Roots.LithBoonkkampIJzerman(3,1)) ≈ α

    @test solve(ZeroProblem((f,fp, fpp), x₀), Roots.LithBoonkkampIJzerman(1,2)) ≈ α
    @test solve(ZeroProblem((f,fp, fpp), x₀), Roots.LithBoonkkampIJzerman(2,2)) ≈ α

    @test solve(ZeroProblem((f,fp, fpp, fppp), x₀), Roots.LithBoonkkampIJzerman(1,3)) ≈ α
    @test solve(ZeroProblem((f,fp, fpp, fppp), x₀), Roots.LithBoonkkampIJzerman(2,3)) ≈ α

    @test solve(ZeroProblem((f,fp, fpp, fppp, fpppp), x₀), Roots.LithBoonkkampIJzerman(1,4)) ≈ α
    @test solve(ZeroProblem((f,fp, fpp, fppp, fpppp), x̃₀), Roots.LithBoonkkampIJzerman(2,4)) ≈ α # needs closer

    # bracketing
    @test solve(ZeroProblem((f,fp), (1,2)), Roots.LithBoonkkampIJzermanBracket()) ≈ α

end


struct CronbachAlpha{T <: Real}
    alpha::T
    dropped::Vector{T}
end

function Base.show(io::IO, x::CronbachAlpha)
    @printf(io, "Cronbach's alpha for all items: %.4f\n", x.alpha)
    isempty(x.dropped) && return
    println(io, "\nCronbach's alpha if an item is dropped:")
    for (idx, val) in enumerate(x.dropped)
        @printf(io, "item %i: %.4f\n", idx, val)
    end
end

"""
    cronbachalpha(covmatrix::AbstractMatrix{<:Real})

Calculate Cronbach's alpha (1951) from a covariance matrix `covmatrix` according to
the [formula](https://en.wikipedia.org/wiki/Cronbach%27s_alpha):

```math
\\rho = \\frac{k}{k-1} (1 - \\frac{\\sum^k_{i=1} \\sigma^2_i}{\\sum_{i=1}^k \\sum_{j=1}^k \\sigma_{ij}})
```

where ``k`` is the number of items, i.e. columns, ``\\sigma_i^2`` the item variance,
and ``\\sigma_{ij}`` the inter-item covariance.

Returns a `CronbachAlpha` object that holds:

* `alpha`: the Cronbach's alpha score for all items, i.e. columns, in `covmatrix`; and
* `dropped`: a vector giving Cronbach's alpha scores if a specific item,
  i.e. column, is dropped from `covmatrix`.

# Example
```jldoctest
julia> using StatsBase

julia> cov_X = [10 6 6 6;
                6 11 6 6;
                6 6 12 6;
                6 6 6 13];

julia> cronbachalpha(cov_X)
Cronbach's alpha for all items: 0.8136

Cronbach's alpha if an item is dropped:
item 1: 0.7500
item 2: 0.7606
item 3: 0.7714
item 4: 0.7826
```
"""
function cronbachalpha(covmatrix::AbstractMatrix{<:Real})
    isposdef(covmatrix) || throw(ArgumentError("Covariance matrix must be positive definite."))
    k = size(covmatrix, 2)
    k > 1  || throw(ArgumentError("Covariance matrix must have more than one column."))
    v = vec(sum(covmatrix, dims=1))
    σ = sum(v)
    for i in axes(v, 1)
        v[i] -= covmatrix[i, i]
    end
    σ_diag = sum(i -> covmatrix[i, i], 1:k)

    alpha = k * (1 - σ_diag / σ) / (k - 1)
    if k > 2
        dropped = typeof(alpha)[(k - 1) * (1 - (σ_diag - covmatrix[i, i]) / (σ - 2*v[i] - covmatrix[i, i])) / (k - 2)
                                for i in 1:k]
    else
        # if k = 2 do not produce dropped; this has to be also
        # correctly handled in show
        dropped = Vector{typeof(alpha)}()
    end
    return CronbachAlpha(alpha, dropped)
end


using Distributions, Random, Plots, LaTeXStrings; pyplot()

seed = 1
N = 100
lamGrid = 0.01:0.01:0.99

theorM(lam) = mean(Uniform(0,2*lam*(1-lam)))
estM(lam) = mean(rand(Uniform(0,2*lam*(1-lam)),N))

function estM(lam,seed)
    Random.seed!(seed)
    estM(lam)
end

trueM = theorM.(lamGrid)
estM0 = estM.(lamGrid)
estMCRN = estM.(lamGrid,seed)

plot(lamGrid,trueM,
	c=:black, label="Expected curve")
plot!(lamGrid,estM0,
	c=:blue, label="No CRN estiamte")
plot!(lamGrid,estMCRN,
	c=:red, label="CRN estimate",
	xlims=(0,1), ylims=(0,0.4), xlabel=L"\lambda", ylabel = "Mean")


using NLSolvers, StaticArrays, Test
@testset "mixed optimization problems" begin
function theta(x)
   if x[1] > 0
       return atan(x[2] / x[1]) / (2.0 * pi)
   else
       return (pi + atan(x[2] / x[1])) / (2.0 * pi)
   end
end
f(x) = 100.0 * ((x[3] - 10.0 * theta(x))^2 + (sqrt(x[1]^2 + x[2]^2) - 1.0)^2) + x[3]^2

function f∇f!(x, ∇f)
    if !(∇f==nothing)
        if ( x[1]^2 + x[2]^2 == 0 )
            dtdx1 = 0;
            dtdx2 = 0;
        else
            dtdx1 = - x[2] / ( 2 * pi * ( x[1]^2 + x[2]^2 ) );
            dtdx2 =   x[1] / ( 2 * pi * ( x[1]^2 + x[2]^2 ) );
        end
        ∇f[1] = -2000.0*(x[3]-10.0*theta(x))*dtdx1 +
            200.0*(sqrt(x[1]^2+x[2]^2)-1)*x[1]/sqrt( x[1]^2+x[2]^2 );
        ∇f[2] = -2000.0*(x[3]-10.0*theta(x))*dtdx2 +
            200.0*(sqrt(x[1]^2+x[2]^2)-1)*x[2]/sqrt( x[1]^2+x[2]^2 );
        ∇f[3] =  200.0*(x[3]-10.0*theta(x)) + 2.0*x[3];
    end

    fx = f(x)
    objective_return(fx, ∇f)
end

function f∇f(x, ∇f)
    if !(∇f == nothing)
        ∇f = similar(x)
    end
    fx, ∇f = f∇f!(gx, x)
    objective_return(fx, ∇f)
end
function f∇fs(x, ∇f)
    if !(∇f == nothing)
        if ( x[1]^2 + x[2]^2 == 0 )
            dtdx1 = 0;
            dtdx2 = 0;
        else
            dtdx1 = - x[2] / ( 2 * pi * ( x[1]^2 + x[2]^2 ) )
            dtdx2 =   x[1] / ( 2 * pi * ( x[1]^2 + x[2]^2 ) )
        end

        s1 = -2000.0*(x[3]-10.0*theta(x))*dtdx1 +
            200.0*(sqrt(x[1]^2+x[2]^2)-1)*x[1]/sqrt( x[1]^2+x[2]^2 )
        s2 = -2000.0*(x[3]-10.0*theta(x))*dtdx2 +
            200.0*(sqrt(x[1]^2+x[2]^2)-1)*x[2]/sqrt( x[1]^2+x[2]^2 )
        s3 = 200.0*(x[3]-10.0*theta(x)) + 2.0*x[3]
        ∇f = @SVector [s1, s2, s3]
    end
    objective_return(f(x), ∇f)
end
obj_inplace = OnceDiffed(f∇f!)
obj_outofplace = OnceDiffed(f∇f)
obj_static = OnceDiffed(f∇fs)

x0 = [-1.0, 0.0, 0.0]
xopt = [1.0, 0.0, 0.0]
x0s = @SVector [-1.0, 0.0, 0.0]

println("Starting  from: ", x0)
println("Targeting     : ", xopt)

res = minimize(obj_inplace, copy(x0), NelderMead(), MinOptions())
print("NN  $(summary(NelderMead()))         ")
@printf("%2.2e  %2.2e %2.2e %d\n",  norm(res.info.minimizer-xopt,Inf), norm(res.info.nm_obj, Inf), res.info.minimum, res.info.iter)
res = minimize(obj_inplace, copy(x0), SimulatedAnnealing(), MinOptions())
print("NN  $(summary(SimulatedAnnealing()))         ")
@printf("%2.2e  %2.2e %d\n",  norm(res.info.minimizer-xopt,Inf),  res.info.minimum, res.info.iter)
res = minimize(obj_inplace, copy(x0), ConjugateGradient(update=HZ()), MinOptions())
print("NN  $(summary(ConjugateGradient(update=HZ())))         ")
@printf("%2.2e  %2.2e %d\n",  norm(res.info.minimizer-xopt,Inf),  res.info.minimum, res.info.iter)
res = minimize(obj_inplace, copy(x0), ConjugateGradient(update=CD()), MinOptions())
print("NN  $(summary(ConjugateGradient(update=CD())))         ")
@printf("%2.2e  %2.2e %d\n",  norm(res.info.minimizer-xopt,Inf),  res.info.minimum, res.info.iter)
res = minimize(obj_inplace, copy(x0), ConjugateGradient(update=HS()), MinOptions())
print("NN  $(summary(ConjugateGradient(update=HS())))         ")
@printf("%2.2e  %2.2e %d\n",  norm(res.info.minimizer-xopt,Inf),  res.info.minimum, res.info.iter)
res = minimize(obj_inplace, copy(x0), ConjugateGradient(update=FR()), MinOptions())
print("NN  $(summary(ConjugateGradient(update=FR())))         ")
@printf("%2.2e  %2.2e %d\n",  norm(res.info.minimizer-xopt,Inf),  res.info.minimum, res.info.iter)
res = minimize(obj_inplace, copy(x0), ConjugateGradient(update=PRP()), MinOptions())
print("NN  $(summary(ConjugateGradient(update=PRP(;plus=false))))         ")
@printf("%2.2e  %2.2e %d\n",  norm(res.info.minimizer-xopt,Inf),  res.info.minimum, res.info.iter)
res = minimize(obj_inplace, copy(x0), ConjugateGradient(update=PRP(plus=true)), MinOptions())
print("NN  $(summary(ConjugateGradient(update=PRP(;plus=true))))         ")
@printf("%2.2e  %2.2e %d\n",  norm(res.info.minimizer-xopt,Inf),  res.info.minimum, res.info.iter)
res = minimize(obj_inplace, copy(x0), ConjugateGradient(update=VPRP()), MinOptions())
print("NN  $(summary(ConjugateGradient(update=VPRP())))         ")
@printf("%2.2e  %2.2e %d\n",  norm(res.info.minimizer-xopt,Inf),  res.info.minimum, res.info.iter)
res = minimize(obj_inplace, copy(x0), ConjugateGradient(update=LS()), MinOptions())
print("NN  $(summary(ConjugateGradient(update=LS())))         ")
@printf("%2.2e  %2.2e %d\n",  norm(res.info.minimizer-xopt,Inf),  res.info.minimum, res.info.iter)
res = minimize(obj_inplace, copy(x0), ConjugateGradient(update=DY()), MinOptions())
print("NN  $(summary(ConjugateGradient(update=DY())))         ")
@printf("%2.2e  %2.2e %d\n",  norm(res.info.minimizer-xopt,Inf),  res.info.minimum, res.info.iter)

for _method in (GradientDescent, LBFGS, BFGS, DBFGS, DFP, SR1)
# for _method in (GradientDescent, BFGS, DBFGS, DFP, SR1)
    methodtxt = summary(_method())
    for m in (Inverse(), Direct())
        mtxt = m isa Inverse ? "(inverse): " : "(direct):  "
        if _method == LBFGS && m isa Inverse
            res = minimize!(obj_inplace, copy(x0), LineSearch(_method(m)), MinOptions())
            print("NN! $_method    $mtxt")
            @printf("%2.2e  %2.2e %2.2e  %d\n", norm(res.info.minimizer-xopt,Inf), norm(res.info.∇fz, Inf), res.info.minimum, res.info.iter)
        elseif _method !== LBFGS
            res = minimize(obj_inplace, copy(x0), LineSearch(_method(m)), MinOptions())
            print("NN  $_method    $mtxt")
            @printf("%2.2e  %2.2e %2.2e  %d\n", norm(res.info.minimizer-xopt,Inf), norm(res.info.∇fz, Inf), res.info.minimum, res.info.iter)
            res = minimize!(obj_inplace, copy(x0), LineSearch(_method(m)), MinOptions())
            print("NN! $_method    $mtxt")
            @printf("%2.2e  %2.2e %2.2e  %d\n", norm(res.info.minimizer-xopt,Inf), norm(res.info.∇fz, Inf), res.info.minimum, res.info.iter)
            res = minimize(obj_static, x0s, LineSearch(_method(m)), MinOptions())
            print("NN  $_method(S) $mtxt")
            @printf("%2.2e  %2.2e %2.2e  %d\n", norm(res.info.minimizer-xopt,Inf), norm(res.info.∇fz, Inf), res.info.minimum, res.info.iter)
            if m isa Direct && !(_method == GradientDescent)
                println("Trust region: NWI")
                res = minimize!(obj_inplace, copy(x0), TrustRegion(_method(Direct()), NWI()), MinOptions())
                print("NN! $_method    $mtxt")
                @printf("%2.2e  %2.2e %2.2e  %d\n", norm(res.info.minimizer-xopt,Inf), norm(res.info.∇fz, Inf), res.info.minimum, res.info.iter)
                println("Trust region: NTR")
                res = minimize!(obj_inplace, copy(x0), TrustRegion(_method(Direct()), NTR()), MinOptions())
                print("NN! $_method    $mtxt")
                @printf("%2.2e  %2.2e %2.2e  %d\n", norm(res.info.minimizer-xopt,Inf), norm(res.info.∇fz, Inf), res.info.minimum, res.info.iter)
            end
        end
    end
end
println()

res = minimize(obj_inplace, x0, LineSearch(BFGS(Inverse())), MinOptions())
@test res.info.iter == 30
@printf("NN  BFGS    (inverse): %2.2e  %2.2e %2.2e %d\n", norm(res.info.minimizer-xopt,Inf), norm(res.info.∇fz, Inf), res.info.minimum, res.info.iter)
res = minimize!(obj_inplace, copy(x0), LineSearch(BFGS(Inverse()), Backtracking()), MinOptions())
@test res.info.iter == 30
@printf("NN! BFGS    (inverse): %2.2e  %2.2e %2.2e %d\n",  norm(res.info.minimizer-xopt,Inf), norm(res.info.∇fz, Inf), res.info.minimum, res.info.iter)
res = minimize!(obj_inplace, copy(x0), LineSearch(BFGS(Inverse()), Backtracking(interp=FFQuadInterp())), MinOptions())
@test res.info.iter == 30
@printf("NN! BFGS    (inverse, quad): %2.2e %2.2e %2.2e %d\n",  norm(res.info.minimizer-xopt,Inf), norm(res.info.∇fz, Inf), res.info.minimum, res.info.iter)
res = minimize(obj_static, x0s, LineSearch(BFGS(Inverse())), MinOptions())
@test res.info.iter == 30
@printf("NN  BFGS(S) (inverse): %2.2e %2.2e %2.2e %d\n",  norm(res.info.minimizer-xopt,Inf), norm(res.info.∇fz, Inf), res.info.minimum, res.info.iter)
res = minimize(obj_static, x0s, LineSearch(BFGS(Inverse()), Backtracking(interp=FFQuadInterp())), MinOptions())
@test res.info.iter == 30
@printf("NN  BFGS(S) (inverse, quad): %2.2e %2.2e %2.2e %d\n",  norm(res.info.minimizer-xopt,Inf), norm(res.info.∇fz, Inf), res.info.minimum, res.info.iter)

x0 = rand(3)
x0s = SVector{3}(x0)
println("\nFrom a random point: ", x0)
res = minimize(obj_inplace, copy(x0), NelderMead(), MinOptions())
print("NN  $(summary(NelderMead()))         ")
@printf("%2.2e  %2.2e %2.2e %d\n",  norm(res.info.minimizer-xopt,Inf), norm(res.info.nm_obj, Inf), res.info.minimum, res.info.iter)
res = minimize(obj_inplace, copy(x0), SimulatedAnnealing(), MinOptions())
print("NN  $(summary(SimulatedAnnealing()))         ")
@printf("%2.2e  %2.2e %d\n",  norm(res.info.minimizer-xopt,Inf),  res.info.minimum, res.info.iter)
res = minimize(obj_inplace, copy(x0), ConjugateGradient(), MinOptions())
print("NN  $(summary(ConjugateGradient()))         ")
@printf("%2.2e  %2.2e %d\n",  norm(res.info.minimizer-xopt,Inf),  res.info.minimum, res.info.iter)

for _method in (GradientDescent, LBFGS, BFGS, DBFGS, DFP, SR1)
    methodtxt = summary(_method())
    for m in (Inverse(), Direct())
        mtxt = m isa Inverse ? "(inverse): " : "(direct):  "
        if _method == LBFGS && m isa Inverse
            res = minimize!(obj_inplace, copy(x0), LineSearch(_method(m)), MinOptions())
            print("NN! $_method    $mtxt")
            @printf("%2.2e  %2.2e %2.2e  %d\n", norm(res.info.minimizer-xopt,Inf), norm(res.info.∇fz, Inf), res.info.minimum, res.info.iter)
        elseif _method !== LBFGS
            res = minimize(obj_inplace, copy(x0), LineSearch(_method(m)), MinOptions())
            print("NN  $_method    $mtxt")
            @printf("%2.2e  %2.2e %2.2e %d\n",  norm(res.info.minimizer-xopt,Inf), norm(res.info.∇fz, Inf), res.info.minimum, res.info.iter)
            res = minimize!(obj_inplace, copy(x0), LineSearch(_method(m)), MinOptions())
            print("NN! $_method    $mtxt")
            @printf("%2.2e  %2.2e %2.2e %d\n",  norm(res.info.minimizer-xopt,Inf), norm(res.info.∇fz, Inf), res.info.minimum, res.info.iter)
            res = minimize(obj_static, x0s, LineSearch(_method(m)), MinOptions())
            print("NN  $_method(S) $mtxt")
            @printf("%2.2e  %2.2e %2.2e %d\n",  norm(res.info.minimizer-xopt,Inf), norm(res.info.∇fz, Inf), res.info.minimum, res.info.iter)
            if m isa Direct && !(_method == GradientDescent)
                println("Trust region: NWI")
                res = minimize!(obj_inplace, copy(x0), TrustRegion(_method(Direct()), NWI()), MinOptions())
                print("NN! $_method    $mtxt")
                @printf("%2.2e  %2.2e %2.2e %d\n",  norm(res.info.minimizer-xopt,Inf), norm(res.info.∇fz, Inf), res.info.minimum, res.info.iter)
                println("Trust region: NTR")
                res = minimize!(obj_inplace, copy(x0), TrustRegion(_method(Direct()), NTR()), MinOptions())
                print("NN! $_method    $mtxt")
                @printf("%2.2e  %2.2e %2.2e %d\n",  norm(res.info.minimizer-xopt,Inf), norm(res.info.∇fz, Inf), res.info.minimum, res.info.iter)
            end
        end
    end
end
println()

function himmelblau!(x, ∇f)
    if !(∇f == nothing)
        ∇f[1] = 4.0 * x[1]^3 + 4.0 * x[1] * x[2] -
            44.0 * x[1] + 2.0 * x[1] + 2.0 * x[2]^2 - 14.0
        ∇f[2] = 2.0 * x[1]^2 + 2.0 * x[2] - 22.0 +
            4.0 * x[1] * x[2] + 4.0 * x[2]^3 - 28.0 * x[2]
    end

    fx = (x[1]^2 + x[2] - 11)^2 + (x[1] + x[2]^2 - 7)^2
    objective_return(fx, ∇f)
end

function himmelblaus(x, ∇f)
    fx = (x[1]^2 + x[2] - 11)^2 + (x[1] + x[2]^2 - 7)^2
    if !(∇f == nothing)
        ∇f1 = 4.0 * x[1]^3 + 4.0 * x[1] * x[2] -
            44.0 * x[1] + 2.0 * x[1] + 2.0 * x[2]^2 - 14.0
        ∇f2 = 2.0 * x[1]^2 + 2.0 * x[2] - 22.0 +
            4.0 * x[1] * x[2] + 4.0 * x[2]^3 - 28.0 * x[2]
        ∇f = @SVector([∇f1, ∇f2])
    end
    objective_return(fx, ∇f)
end

function himmelblau(x, ∇f)
    g = ∇f == nothing ? ∇f : similar(x)

    return himmelblau!(x, g)
end

him_inplace = OnceDiffed(himmelblau!)
him_static = OnceDiffed(himmelblaus)
him_outofplace = OnceDiffed(himmelblau)

println("\nHimmelblau function")
x0 = [3.0, 1.0]
x0s = SVector{2}(x0)
minimizers = [[3.0,2.0],[-2.805118,3.131312],[-3.779310,-3.283186],[3.584428,-1.848126]]
for _method in (GradientDescent, LBFGS, BFGS, DBFGS, DFP, SR1)
# for _method in (GradientDescent, BFGS, DBFGS, DFP, SR1)
    methodtxt = summary(_method())
    for m in (Inverse(), Direct())
        mtxt = m isa Inverse ? "(inverse): " : "(direct):  "
        if _method == LBFGS && m isa Inverse
            res = minimize!(him_inplace, copy(x0), LineSearch(_method(m)), MinOptions())
            print("NN! $_method    $mtxt")
            @printf("%2.2e  %2.2e %2.2e  %d\n", minimum([norm(res.info.minimizer-xopt,Inf) for xopt in minimizers]), norm(res.info.∇fz, Inf), res.info.minimum, res.info.iter)
        elseif _method !== LBFGS
            res = minimize(him_outofplace, copy(x0), LineSearch(_method(m)), MinOptions())
            print("NN  $_method    $mtxt")
            @printf("%2.2e  %2.2e %2.2e  %d\n", minimum([norm(res.info.minimizer-xopt,Inf) for xopt in minimizers]), norm(res.info.∇fz, Inf), res.info.minimum, res.info.iter)
            res = minimize!(him_inplace, copy(x0), LineSearch(_method(m)), MinOptions())
            print("NN! $_method    $mtxt")
            @printf("%2.2e  %2.2e %2.2e  %d\n", minimum([norm(res.info.minimizer-xopt,Inf) for xopt in minimizers]), norm(res.info.∇fz, Inf), res.info.minimum, res.info.iter)
            res = minimize(him_static, x0s, LineSearch(_method(m)), MinOptions())
            print("NN  $_method(S) $mtxt")
            @printf("%2.2e  %2.2e %2.2e  %d\n", minimum([norm(res.info.minimizer-xopt,Inf) for xopt in minimizers]), norm(res.info.∇fz, Inf), res.info.minimum, res.info.iter)
            if m isa Direct && !(_method == GradientDescent)
                println("Trust region: NWI")
                res = minimize!(him_inplace, copy(x0), TrustRegion(_method(Direct()), NWI()), MinOptions())
                print("NN! $_method    $mtxt")
                @printf("%2.2e  %2.2e %2.2e  %d\n", minimum([norm(res.info.minimizer-xopt,Inf) for xopt in minimizers]), norm(res.info.∇fz, Inf), res.info.minimum, res.info.iter)
                println("Trust region: NTR")
                res = minimize!(him_inplace, copy(x0), TrustRegion(_method(Direct()), NTR()), MinOptions())
                print("NN! $_method    $mtxt")
                @printf("%2.2e  %2.2e %2.2e  %d\n", minimum([norm(res.info.minimizer-xopt,Inf) for xopt in minimizers]), norm(res.info.∇fz, Inf), res.info.minimum, res.info.iter)
            end
        end
    end
end
println()
println()
xrand = rand(2)
xrands = SVector{2}(xrand)
println("\nFrom a random point: ", xrand)

for _method in (GradientDescent, LBFGS, BFGS, DBFGS, SR1)
# for _method in (GradientDescent, BFGS, DBFGS, DFP, SR1)
    methodtxt = summary(_method())
    for m in (Inverse(), Direct())
        mtxt = m isa Inverse ? "(inverse): " : "(direct):  "
        if _method == LBFGS && m isa Inverse
            res = minimize!(him_inplace, copy(x0), LineSearch(_method(m)), MinOptions())
            print("NN! $_method    $mtxt")
            @printf("%2.2e  %2.2e %2.2e  %d\n", minimum([norm(res.info.minimizer-xopt,Inf) for xopt in minimizers]), norm(res.info.∇fz, Inf), res.info.minimum, res.info.iter)
        elseif _method !== LBFGS
            res = minimize(him_outofplace, copy(xrand), LineSearch(_method(m)), MinOptions())
            print("NN  $_method    $mtxt")
            @printf("%2.2e  %2.2e %2.2e  %d\n", minimum([norm(res.info.minimizer-xopt,Inf) for xopt in minimizers]), norm(res.info.∇fz, Inf), res.info.minimum, res.info.iter)
            res = minimize!(him_inplace, copy(xrand), LineSearch(_method(m)), MinOptions())
            print("NN! $_method    $mtxt")
            @printf("%2.2e  %2.2e %2.2e  %d\n", minimum([norm(res.info.minimizer-xopt,Inf) for xopt in minimizers]), norm(res.info.∇fz, Inf), res.info.minimum, res.info.iter)
            res = minimize(him_static, xrands, LineSearch(_method(m)), MinOptions())
            print("NN  $_method(S) $mtxt")
            @printf("%2.2e  %2.2e %2.2e  %d\n", minimum([norm(res.info.minimizer-xopt,Inf) for xopt in minimizers]), norm(res.info.∇fz, Inf), res.info.minimum, res.info.iter)
            if m isa Direct && !(_method == GradientDescent)
                println("Trust region: NWI")
                res = minimize!(him_inplace, copy(xrand), TrustRegion(_method(Direct()), NWI()), MinOptions())
                print("NN! $_method    $mtxt")
                @printf("%2.2e  %2.2e %2.2e  %d\n", minimum([norm(res.info.minimizer-xopt,Inf) for xopt in minimizers]), norm(res.info.∇fz, Inf), res.info.minimum, res.info.iter)
                println("Trust region: NTR")
                res = minimize!(him_inplace, copy(xrand), TrustRegion(_method(Direct()), NTR()), MinOptions())
                print("NN! $_method    $mtxt")
                @printf("%2.2e  %2.2e %2.2e  %d\n", minimum([norm(res.info.minimizer-xopt,Inf) for xopt in minimizers]), norm(res.info.∇fz, Inf), res.info.minimum, res.info.iter)
            end
        end
    end
end
println()

end