using PyPlot

pospart(x) = ( x > 0 ? x : zero(x) )

function K2(x, y, a, b)
    pt = [a, (a+b)/2, b]
    ℓ2 = (x-pt[1])*(x-pt[3])/((pt[2]-pt[1])*(pt[2]-pt[3]))
    ℓ3 = (x-pt[1])*(x-pt[2])/((pt[3]-pt[1])*(pt[3]-pt[2]))
    Qπ = pospart(pt[2]-y)^2 * ℓ2 / 2 + (b-y)^2 * ℓ3 / 2
    return Qπ - pospart(x-y)^2/2
end

a = -1.0
b = 1.0
N = 200
x = LinRange(a, b, N+1)
y = LinRange(a, b, N+1)
z = Float64[ K2(x[k], y[j], a, b) for j=1:N+1, k=1:N+1 ]
fig = figure(1)
surf(x, y, z, cstride=4, rstride=4, cmap="jet", linewidth=0.25)
xlabel(L"x")
ylabel(L"y")
ax = gca()
ax[:view_init](elev=40,azim=-100)
savefig("Quadratic_PeanoK_3d.pdf")

figure(2)
contour(x, y, z, 12)
colorbar()
xlabel(L"x")
ylabel(L"y")
grid(true)
savefig("Quadratic_PeanoK_contour.pdf")


@testset "The Symmetric Matrices" begin
    #
    # Despite checking both cases of Eucliean Manifolds, this can serve as a
    # prototype testcase suite for your manifold
    #
    M = Symmetric(2)
    v1 = [1. 0.;0. 1.]
    v2 = [2. 1.; 1. 2.]
    d1 = [1. 1.;1. 1.]
    d2 = [1. 0.; 0. 0.]

    x1 = SymPoint(v1)
    x2 = SymPoint(v2)
    ξ1 = SymTVector(d1)
    ξ2 = SymTVector(d2)

    @test getValue(x1) == v1
    @test getValue(ξ1) == d1

    @test distance(M,x1,x2) == norm(v1-v2)

    @test dot(M,x1,ξ1,ξ2) == dot(d1,d2)

    @test exp(M,x1, ξ1) == SymPoint(v1 + d1)
    @test exp(M,x2, ξ2) == SymPoint(v2 + d2)

    @test log(M,x1,x2) == SymTVector(v2-v1)

    @test manifoldDimension(M) == 3
    @test manifoldDimension(M) == manifoldDimension(x1)

    @test norm(M,x1,ξ1) == norm(d1)

    @test parallelTransport(M,x1,x2,ξ1) == ξ1

    @test validateMPoint(M,x1)
    @test_throws DomainError !validateMPoint(M, SymPoint([1. 0.; 1. 1.]))
    @test_throws DomainError !validateMPoint(M, SymPoint([1. 0. 0.; 0. 1. 0.; 0. 0. 1.]))
    @test validateTVector(M,x1,ξ1)
    @test_throws DomainError validateTVector(M,x1,SymTVector([1. 0.; 1. 1.]))
    @test_throws DomainError validateTVector(M,x1,SymTVector([1. 0. 0.; 0. 1. 0.; 0. 0. 1.]))

    @test typeofTVector(typeof(x1)) == SymTVector{Float64}
    @test typeofMPoint(typeof(ξ1)) == SymPoint{Float64}
    @test typeofTVector(x1) == SymTVector{Float64}
    @test typeofMPoint(ξ1) == SymPoint{Float64}

    @test typicalDistance(M) == sqrt(2)

    @test zeroTVector(M,x1) == SymTVector(zeros(2,2))

    @test "$M" == "The Manifold of 2-by-2 symmetric matrices."
    @test "$(x1)" == "Sym([1.0 0.0; 0.0 1.0])"
    @test "$(ξ1)" == "SymT([1.0 1.0; 1.0 1.0])"
end


## sigmoidal functions

export logistic, logistic_prime, weibull, weibull_prime

logistic(x) = 1/(1 + exp(-x))
logistic_prime(x) = exp(-x)/(1 + exp(-x))^2
weibull(x, coef, scale, shape) = (1 - exp(-((x/scale)^shape)))*coef
weibull_prime(x, coef, scale, shape) = coef*(shape/scale)*((x/scale)^(shape-1))*exp(-((x/scale)^shape))


# ---
# title: 633. Sum of Square Numbers
# id: problem633
# author: Indigo
# date: 2021-06-27
# difficulty: Medium
# categories: Math
# link: <https://leetcode.com/problems/sum-of-square-numbers/description/>
# hidden: true
# ---
#
# Given a non-negative integer `c`, decide whether there're two integers `a` and
# `b` such that `a2 + b2 = c`.
#
#
#
# **Example 1:**
#
#
#
#     Input: c = 5
#     Output: true
#     Explanation: 1 * 1 + 2 * 2 = 5
#
#
# **Example 2:**
#
#
#
#     Input: c = 3
#     Output: false
#
#
# **Example 3:**
#
#
#
#     Input: c = 4
#     Output: true
#
#
# **Example 4:**
#
#
#
#     Input: c = 2
#     Output: true
#
#
# **Example 5:**
#
#
#
#     Input: c = 1
#     Output: true
#
#
#
#
# **Constraints:**
#
#   * `0 <= c <= 231 - 1`
#
#
## @lc code=start
using LeetCode

function judge_square_sum(n::Int)
    upper = isqrt(n)
    i = 0
    while i <= upper
        ss = i ^ 2 + upper ^ 2
        ss == n && return true
        ss > n ? (upper -= 1) : (i += 1)
    end
    return false
end
## @lc code=end


"""
    lis(arr::Array{Int}, ::Val{:bs})

# Arguments:
- `arr`: sequence of integers

# Examples/Tests
```julia
julia> lis([3, 10, 2, 1, 20], Val(:bs))
[3, 10, 20]

julia> lis([2, 2, 3], Val(:bs))
[2, 3]
```

# Brief:
Given a sequence of integers, the function finds the longest, strictly increasing, subsequence in that sequence.

# For more information:
https://cp-algorithms.com/sequences/longest_increasing_subsequence.html

# Contributed by:
- [Igor Malheiros](https://github.com/igormalheiros)
"""

function lis(arr::Array{Int}, ::Val{:bs})
    len = length(arr)
    memo = ones(Int, len)
    p = ones(Int, len)

    lis_arr = Int[]

    len == 0 && return lis_arr # if `arr` is empty

    lis_len = 1
    memo[1] = arr[1]

    for i = 2:len
        # binary search in current `memo`, it uses @view and slicing for in-place verification
        p[i] = searchsortedfirst((@view memo[1:lis_len]), arr[i])
        # if arr[i] is greater than the greatest element in `memo`, it increases `lis_len`
        lis_len = max(lis_len, p[i])
        # updates `memo`
        memo[p[i]] = arr[i]
    end

    # Restoring
    last_pos = lis_len
    for i = len:-1:1
        if p[i] == last_pos
            push!(lis_arr, arr[i])
            last_pos -= 1
        end
    end

    reverse!(lis_arr)

    return lis_arr
end


module JacobiAlgorithm

    struct EquilibriumProblem{T<:Real}
        nbz::Int64
        e_z::Function
        pmin::T
        pmax::T
        q_z::Vector{T}
        p0_z::Vector{T}
        function EquilibriumProblem(nbz::Int64,e_z::Function,pmin::T,pmax::T,q_z::Vector{T},p0_z::Vector{T})  where {T<:Real}
            this = new{T}(nbz,e_z,pmin,pmax,q_z,p0_z) #,NaN,[],[],[],NaN)
        end
    end

    function bisection(f::Function, a::T, b::T;
        tol::T=1e-5, maxiter::Int64=100) where T<:Real
        fa = f(a)
        fa*f(b) <= 0 || error("No real root in [a,b]")
        i = 0
        local c
        while b-a > tol
                i += 1
                i != maxiter || error("Max iteration exceeded")
                c = (a+b)/2
                fc = f(c)
            if fc == 0
                break
            elseif fa*fc > 0
                a = c  # Root is in the right half of [a,b].
                fa = fc
            else
                b = c  # Root is in the left half of [a,b].
            end
        end
        return c
    end

    function coordinate_update(problem::EquilibriumProblem{T}, z::Int64, price_z::Vector{T}) where T<:Real
        old_price = price_z[z]
        f(p) = problem.e_z(setindex!(price_z,p,z))[z] - problem.q_z[z]
        new_price = bisection(f, problem.pmin, problem.pmax,tol=1e-5,maxiter=1000)
        price_z[z] = old_price
        return new_price, new_price - old_price
    end

    function fJ_z(problem::EquilibriumProblem{T}, p_z::Vector{T}) where T<:Real
        nbz = size(p_z,1)
        pp_z = zeros(T, nbz)
        diffp_z = zeros(T, nbz)
        for z = 1:nbz
            pp_z[z], diffp_z[z] = coordinate_update(problem,z,p_z)
        end
        return pp_z, diffp_z
    end

    function fJ_threaded_z(problem::EquilibriumProblem{T}, p_z::Vector{T}) where T<:Real
        nbz = size(p_z,1)
        pp_z = zeros(T, nbz)
        diffp_z = zeros(T, nbz)
        Threads.@threads for z = 1:nbz
            p_z_safe = deepcopy(p_z)
            pp_z[z], diffp_z[z] = coordinate_update(problem,z,p_z_safe)
        end
        return pp_z, diffp_z
    end

    function solve(problem::EquilibriumProblem{T};
        maxit::Int64 = 10_000,
        method::Symbol = :jacobian,
        valtol::T = 1e-5,
        steptol::T = 1e-9,
        output::Int64 = 0) where T<:Real

        code = 0
        start_time = time()
        delta_z = zeros(T,problem.nbz)
        diffp_z = zeros(T,problem.nbz)
        iterations = 0
        p_z = deepcopy(problem.p0_z)
        if method != :jacobian
            error("Method not implemented")
        end
        for i = 1:maxit
            p_z, diffp_z = fJ_z(problem, p_z)
            delta_z = problem.e_z(p_z) .- problem.q_z'
            if output>1
                println("p = $p_z")
            end
            if maximum(abs.(delta_z)) < valtol
                code = 0
                iterations = i
                break
            end
            if maximum(abs.(diffp_z)) < steptol
                code = 1
                iterations = i
                break
            end
            code = 2
        end
        comp_time = time() - start_time
        if output > 0
            println("$(String(method)) method converged in $iterations iterations.")
            println("Value of p = $(p_z)")
            println("Value of p' - p = $(diffp_z)")
            println("Value of e(p)-q = $(delta_z)")
            println("Time elapsed: $comp_time")
            println("Code: $code")
        end
        return code
    end

    function solve_threaded(problem::EquilibriumProblem{T};
        maxit::Int64 = 10_000,
        method::Symbol = :jacobian,
        valtol::T = 1e-5,
        steptol::T = 1e-9,
        output::Int64 = 0) where T<:Real

        code = 0
        start_time = time()
        delta_z = zeros(T,problem.nbz)
        diffp_z = zeros(T,problem.nbz)
        iterations = 0
        p_z = deepcopy(problem.p0_z)
        if method != :jacobian
            error("Method not implemented")
        end
        for i = 1:maxit
            p_z, diffp_z = fJ_threaded_z(problem, p_z)
            delta_z = problem.e_z(p_z) .- problem.q_z'
            if output>1
                println("p = $p_z")
            end
            if maximum(abs.(delta_z)) < valtol
                code = 0
                iterations = i
                break
            end
            if maximum(abs.(diffp_z)) < steptol
                code = 1
                iterations = i
                break
            end
            code = 2
        end
        comp_time = time() - start_time
        if output > 0
            println("$(String(method)) method converged in $iterations iterations.")
            println("Value of p = $(p_z)")
            println("Value of p' - p = $(diffp_z)")
            println("Value of e(p)-q = $(delta_z)")
            println("Time elapsed: $comp_time")
            println("Code: $code")
        end
        return code
    end
end


# this is a little Monte Carlo exercise that illustrates that
# the OLS estimator is biased and inconsistent when errors are
# correlated with regressors, but that the IV estimator is consistent
using Econometrics, Plots, LinearAlgebra
function main()
reps = 1000 # number of Monte Carlo reps.
betaols = zeros(reps,2)
betaiv = zeros(reps,2)
n = 1000 # sample size

# covariance of X, W, e
cor_X_W = 0.2  # experiment with lowering or raising this: quality of instrument
cor_X_e = 0.5  # try setting this to zero to observe inefficiency of IV
sig = [
      1.0       cor_X_W     cor_X_e;
      cor_X_W   1.0         0.0;
     cor_X_e    0.0         1.0]
truebeta = [1, 2] # true beta
p = cholesky(sig).U
for i = 1:reps
	XWE = randn(n,3)*p
	e = XWE[:,3:3]
	w = [ones(n) XWE[:,2:2]]
	x = [ones(n) XWE[:,1:1]]
	y = x*truebeta + e
	# OLS
	betaols[i,:] = (x\y)'
	# IV
	betaiv[i,:] = (inv(w'*x)*w'*y)'
end

p1 = npdensity(betaols[:,2])
plot!(p1,title="OLS")
p2 = npdensity(betaiv[:,2])
plot!(p2,title="IV")
p = plot(p1,p2,layout=(2,1))
#savefig("olsiv.png")
gui()
println("true betas are ", truebeta)
println("OLS results")
dstats(betaols, short=true)
println("IV results")
dstats(betaiv, short=true)
return p
end
main()


#find row based on the code, to improve this function
# and the find_column could do a hi_lo funtion
# and then call that for both find_row and find_column

function find_row(code)
  row_code = code[1:7]
  lo_n = 0
  hi_n = 127

  count = 1

  for i in row_code
     #find the bounds of the range depending if we can the upper or lower half
     if i =='F'
        hi_n = lo_n + div(hi_n+1 - lo_n,2) - 1
        if count == 7
           return hi_n
        end
     else
        lo_n = lo_n + div(hi_n+1 - lo_n,2)
        if count == 7
           return lo_n
        end
     end

    count += 1

  end

  return 1
end

function find_column(code)
  col_code = code[8:end]

  lo_n = 0
  hi_n = 7

  count = 1

  for i in col_code

     if i =='L'
        hi_n = lo_n + div(hi_n+1 - lo_n,2) - 1
        if count == 3
           return hi_n
        end
     else
        lo_n = lo_n + div(hi_n+1 - lo_n,2)
        if count == 3
           return lo_n
        end
     end

    count += 1

  end

  return 1
end

using DelimitedFiles

# reading the input
data = readdlm("input_day5")

max_id = 0

for temp_code in data
   r = find_row(temp_code)
   c = find_column(temp_code)

   #calculate id
   id = r*8 + c

   #find max id
   if id > max_id
     global max_id = id
   end

end

println(max_id)


using OrdinaryDiffEq
using DiffEqCallbacks
using GlobalSensitivity
using Distributions
using Plots

function sir_ode!(du,u,p,t)
    (S,I,R) = u
    (β,c,γ) = p
    N = S+I+R
    @inbounds begin
        du[1] = -β*c*I/N*S
        du[2] = β*c*I/N*S - γ*I
        du[3] = γ*I
    end
    nothing
end;

tmax = 10000.0
tspan = (0.0,tmax);

cb_ss = TerminateSteadyState();

N = 1000.0;
u0 = [990.0,10.0,0.0];
p = [0.05,10.0,0.25]; # β,c,γ

n_samples = 1000 # Number of samples
# Parameters are β, c, γ, I₀
lb = [0.01, 5.0, 0.1, 1.0]
ub = [0.1, 20.0, 1.0, 50.0]
n_params = 4;

prob_ode = ODEProblem(sir_ode!,u0,tspan,p);

f1 = function(pu0)
  p = pu0[1:3]
  I0 = pu0[4]
  u0 = [N-I0,I0,0.0]
  prob = remake(prob_ode;p=p,u=u0)
  sol = solve(prob, ROS34PW3(),callback=cb_ss)
  [maximum(sol[2,:]), sol.t[argmax(sol[2,:])], sol[end][3]]
end;

m_morris = gsa(f1, Morris(num_trajectory=n_samples), [[lb[i],ub[i]] for i in 1:n_params]);

m_morris.means

m_morris.variances

m_sobol = gsa(f1, Sobol(), [[lb[i],ub[i]] for i in 1:n_params],N=n_samples);

m_sobol.ST

m_sobol.S1

m_regression = gsa(f1, RegressionGSA(rank=true), [[lb[i],ub[i]] for i in 1:n_params]; samples = n_samples);

m_regression.partial_correlation

m_regression.partial_rank_correlation

m_efast = gsa(f1, eFAST(), [[lb[i],ub[i]] for i in 1:n_params]; n = n_samples);

m_efast.ST

m_efast.S1

pf1 = function (pu0)
  p = pu0[1:3,:]
  I0 = pu0[4,:]
  prob_func(prob,i,repeat) = remake(prob;p=p[:,i],u=[N-I0[i],I0[i],0.0])
  ensemble_prob = EnsembleProblem(prob_ode,prob_func=prob_func)
  sol = solve(ensemble_prob,ROS34PW3(),EnsembleThreads();trajectories=size(p,2))
  out = zeros(3,size(p,2))
  for i in 1:size(p,2)
    out[1,i] = maximum(sol[i][2,:])
    out[2,i] = sol[i].t[argmax(sol[i][2,:])]
    out[3,i] = sol[i][end][3]
  end
  out
end;

m_efast_parallel = gsa(pf1, eFAST(), [[lb[i],ub[i]] for i in 1:n_params]; n = n_samples, batch = true);


# # [Time integration](@id Time-integration)

# Time integration methods for the numerical solution of Ordinary Differential
# Equations (ODEs), also called timesteppers, can be of different nature and
# flavor (e.g., explicit, semi-implicit, single-stage, multi-stage, single-step,
# multi-step, single-rate, multi-rate, etc). ClimateMachine supports several
# of them. Before showing the different nature of some of these methods, let us
# introduce some common notation.

# A commonly used notation for Initial Value Problems (IVPs) is:

# ```math
# \begin{align}
#     \frac{\mathrm{d} \boldsymbol{q}}{ \mathrm{d} t} &= \mathcal{T}(t, \boldsymbol{q}),\\
#     \boldsymbol{q}(t_0) &= \boldsymbol{q_0},
# \end{align}
# ```

# where ``\boldsymbol{q}`` is an unknown function (vector in most of our cases)
# of time ``t``, which we would like to approximate, and at the initial time ``t_0``
# the corresponding initial value ``\boldsymbol{q}_0`` is given.

# The given general formulation, is suitable for single-step explicit schemes.
# Generally, the equation can be represented in the following canonical form:

# ```math
# \begin{align}
#      \dot {\boldsymbol{q}} + \mathcal{F}(t, \boldsymbol{q}) &= \mathcal{G}(t, \boldsymbol{q}),
# \end{align}
# ```

# where we have used ``\dot {\boldsymbol{q}} = d \boldsymbol{q} / dt``.
# We refer to the term ``\mathcal{G}``
# as the right-hand-side (RHS) or explicit term, and to the spatial terms of
# ``\mathcal{F}`` as the left-hand-side (LHS) or implicit term.


"""
    logistic2_bidir(u₀, c_xy, c_yx, r₁, r₂, σ_xy, σ_yx)

Equations of motion for a bidirectional logistic model for the chaotic
population dynamics of two interacting species. This system is from [1],
and is given by

```math
\\begin{align}
x(t+1) &= r_1 f_{yx}^{t}(1 - f_{yx}^{t}) \\
y(t+1) &= r_2 f_{xy}^{t}(1 - f_{xy}^{t}) \\
f_{xy}^t &= \\dfrac{y(t) + c_{xy}(x(t) + \\sigma_{xy} \\xi_{xy}^t )}{1 + c_{xy} (1 + \\sigma_{xy} )} \\
f_{yx}^t &= \\dfrac{x(t) + c_{yx}(y(t) + \\sigma_{yx} \\xi_{yx}^t )}{1 + c_{yx} (1 + \\sigma_{yx} )},
\\end{align}
```

where the coupling strength ``c_{xy}`` controls how strongly species ``x`` influences species
``y``, and vice versa for ``c_{yx}``. To simulate time-varying influence of unobserved
processes, we use the dynamical noise terms ``\\xi_{xy}^t`` and ``\\xi_{yx}^t``, drawn from a
uniform distribution with support on ``[0, 1]``. If ``\\sigma_{xy} > 0``, then the influence
of ``x`` on ``y`` is masked by dynamical noise equivalent to ``\\sigma_{xy} \\xi_{xy}^{t}`` at
the ``t``-th iteration of the map, and vice versa for ``\\sigma_{yx}``.

## References

1. Diego, David, Kristian Agasøster Haaga, and Bjarte Hannisdal. "Transfer entropy computation
    using the Perron-Frobenius operator." Physical Review E 99.4 (2019): 042212.
"""
function eom_logistic2_bidir(dx, x, p, n)

    # c_xy is the coupling from x to y
    # c_yx is the coupling from y to x
    # σ_yx is the dynamical noise from y to x
    # σ_xy is the dynamical noise from y to x
    c_xy, c_yx, r₁, r₂, σ_xy, σ_yx = (p...,)

    ξ₁ = rand() # random number from flat distribution on [0, 1]
    ξ₂ = rand() # random number from flat distribution on [0, 1]
    x, y = x[1], x[2]

    f_xy = (y +  c_xy*(x + σ_xy*ξ₁) ) / (1 + c_xy*(1+σ_xy))
    f_yx = (x +  c_yx*(y + σ_yx*ξ₂) ) / (1 + c_yx*(1+σ_yx))

    dx[1] = r₁ * (f_yx) * (1 - f_yx)
    dx[2] = r₂ * (f_xy) * (1 - f_xy)
    return
end

function logistic2_bidir(u₀, c_xy, c_yx, r₁, r₂, σ_xy, σ_yx)
    p = [c_xy, c_yx, r₁, r₂, σ_xy, σ_yx]
    DiscreteDynamicalSystem(eom_logistic2_bidir, u₀, p)
end

"""
    logistic2_bidir(;u₀ = rand(2), c_xy = 0.1, c_yx = 0.1,
        r₁ = 3.78, r₂ = 3.66, σ_xy = 0.05, σ_yx = 0.05)

A bidirectional logistic model for the chaotic population dynamics of two interacting
species [1].

## Equations of motion

The equations of motion are

```math
\\begin{align}
x(t+1) &= r_1 f_{yx}^{t}(1 - f_{yx}^{t}) \\\\
y(t+1) &= r_2 f_{xy}^{t}(1 - f_{xy}^{t}) \\\\
f_{xy}^t &= \\dfrac{y(t) + c_{xy}(x(t) + \\sigma_{xy} \\xi_{xy}^t )}{1 + c_{xy} (1 + \\sigma_{xy} )} \\\\
f_{yx}^t &= \\dfrac{x(t) + c_{yx}(y(t) + \\sigma_{yx} \\xi_{yx}^t )}{1 + c_{yx} (1 + \\sigma_{yx} )},
\\end{align}
```

where the coupling strength ``c_{xy}`` controls how strongly species ``x`` influences species
``y``, and vice versa for ``c_{yx}``. To simulate time-varying influence of unobserved
processes, we use the dynamical noise terms ``\\xi_{xy}^t`` and ``\\xi_{yx}^t``, drawn from a
uniform distribution with support on ``[0, 1]``. If ``\\sigma_{xy} > 0``, then the influence
of ``x`` on ``y`` is masked by dynamical noise equivalent to ``\\sigma_{xy} \\xi_{xy}^{t}`` at
the ``t``-th iteration of the map, and vice versa for ``\\sigma_{yx}``.

## References

1. Diego, David, Kristian Agasøster Haaga, and Bjarte Hannisdal. "Transfer entropy computation
    using the Perron-Frobenius operator." Physical Review E 99.4 (2019): 042212.
"""
logistic2_bidir(;u₀ = rand(2), c_xy = 0.1, c_yx = 0.1,
    r₁ = 3.78, r₂ = 3.66, σ_xy = 0.05, σ_yx = 0.05) =
    logistic2_bidir(u₀, c_xy, c_yx, r₁, r₂, σ_xy, σ_yx)


# coding: utf-8

# In[1]:

#coin change is a straight forward dynamic programming problem
#you are given one note and a set of coins of different values
#assuming you have infinite supply of coins of different values
#we need to compute different ways of making change
#the order of the coins doesnt matter in this case
#more details can be found in the following link
# https://www.geeksforgeeks.org/coin-change-dp-7/
#the script can also be done in recursion
# https://github.com/je-suis-tm/recursion-and-dynamic-programming/blob/master/coin%20change%20recursion.jl

# In[2]:

#to solve this problem via tabulation
#we divide one big problem into two sub problems
#the case where coin of η value is excluded in the solutions
#and the case where at least one coin of η value is included
function coin_change(num,choice)

    #create matrix (num+1)*length(choice)
    #the raison d'être is the computation starts from 0 to num
    #0 is when num is perfectly substituted by coins
    tabulation=[[0 for _ in 1:length(choice)] for _ in 1:num+1]

    #initialize
    #when the remain value happens to be η
    #one solution is found
    for i in 1:length(choice)

        tabulation[1][i]=1

    end

    #since we initialize the null case
    #the outerloop starts from 2
    #i actually refers to i-1
    for i in 2:num+1

        for j in 1:length(choice)

            #annoying part of julia
            #if index starts at zero
            #will be a lot easier
            if i-choice[j]>=1

                #the case where at least one coin of η value is included
                #we just need the computation where η is deducted
                include=tabulation[i-choice[j]][j]

            else

                include=0

            end

            #the case where coin of η value is excluded in the solutions
            if j>=2

                exclude=tabulation[i][j-1]

            else

                exclude=0

            end

            #two sub problems merge into a big one
            tabulation[i][j]=exclude+include

        end

    end

    #get the final answer
    return tabulation[num+1][length(choice)]

end

# In[3]:

coin_change(10,[1,2,5])


#=
If we list all the natural numbers below 10 that are multiples of 3 or 5, we get 3, 5, 6 and 9. The sum of these multiples is 23.
Find the sum of all the multiples of 3 or 5 below 1000.
=#

function calc()
  filter(x -> x % 3 == 0 || x % 5 == 0, 1:999) |> sum |> println
end
@time calc()


# fitting noisy data to an exponential model
model(xpts, p) = p[1]*exp(-xpts.*p[2])

# some example data
srand(12345)
xpts = linspace(0,10,20)
data = model(xpts, [1.0 2.0]) + 0.01*randn(length(xpts))

beta, r, J = curve_fit(model, xpts, data, [0.5, 0.5])
@assert norm(beta - [1.0, 2.0]) < 0.05

# can also get error estimates on the fit parameters
errors = estimate_errors(beta, r, J)
@assert norm(errors - [0.016, 0.075]) < 0.01


### A Pluto.jl notebook ###
# v0.12.20

using Markdown
using InteractiveUtils

# ╔═╡ ec794b0c-67b1-11eb-23e7-459641028308
begin
	using Markdown
	using InteractiveUtils
	using Test
	using LinearAlgebra
end

# ╔═╡ 47f1758c-67b1-11eb-0a89-27d2595e5912
md"""

## Task \# 8

Compute the norms and condition numbers

#### Norm

```math
||A||^2 = max \frac{||Ax||^2}{||x||^2} = max \frac{ x^T A^T A x }{ x^T x } = \lambda_{max}(A^T A) = \sigma^{2}_{max}
```

```math
||A|| = \sqrt{\sigma^{2}_{max}} = \sigma_{max}
```

```math
||A^{-1}|| = \frac{1}{\sigma_{min}}
```

"""

# ╔═╡ 9aa05ff6-67b5-11eb-0b36-95986372b105
function normCalculator(A::Array{Int})
	AᵀA = A' * A

	A_eigenvals, A_eigenvec = eigen(AᵀA)

	A_normFromEigenSquared = √max(A_eigenvals...)

	A_norm = opnorm(A)

	return (A_normFromEigenSquared, A_norm)
end

# ╔═╡ 4533fdcc-67b3-11eb-30f8-851ef7fc43c9
begin
	A₁ = [
		1 7
		1 1
	]

	A₁normFromEigenSquared, A₁norm = normCalculator(A₁)

	@test round.(A₁normFromEigenSquared, digits=5) == round.(A₁norm, digits=5)

	A₁normFromEigenSquared, A₁norm, cond(A₁)
end

# ╔═╡ bf65578a-67b8-11eb-2627-9139089fcf68
md"""
#### Cond is Inf, because of `` || A_2^{-1} || = \sigma_{min} = 0 ``
"""

# ╔═╡ 1c2ed1ac-67b5-11eb-1e77-67e96dbd2802
begin
	A₂ = [
		1 1
		0 0
	]

	A₂normFromEigenSquared, A₂norm = normCalculator(A₂)

	@test round.(A₂normFromEigenSquared, digits=5) == round.(A₂norm, digits=5)

	A₂normFromEigenSquared, A₂norm, cond(A₂)
end

# ╔═╡ 1568a204-67b9-11eb-3d55-d72928803e5e
md"""
#### Cond is 1, because of `` || A_2 || = || A_2^{-1} || ``
"""

# ╔═╡ ed7d86f6-67b3-11eb-273c-43770e6f299c
begin
	A₃ = [
		1 1
		-1 1
	]

	A₃normFromEigenSquared, A₃norm = normCalculator(A₃)

	@test round.(A₃normFromEigenSquared, digits=5) == round.(A₃norm, digits=5)

	A₃normFromEigenSquared, A₃norm, cond(A₃)
end

# ╔═╡ c3dc9efe-67bd-11eb-0224-af9d70653b73
md"""
### Task \# 13
Estimate the condition number
"""

# ╔═╡ e21e095c-67bd-11eb-12e4-e92b18b0b223
begin
	A₄ = [
		1 1
		1 1.0001
	]
	cond(A₄)
end

# ╔═╡ 67060602-67c2-11eb-046e-dfc437a3dc81
md"""
### Task \#17
Fibonacci matrix and the SVD
"""

# ╔═╡ 832167be-67c2-11eb-1693-ab8ea1a74995
begin
	F = [
		1 1
		1 0
	]
	svd(F)
end

# ╔═╡ 8d7f475a-67c3-11eb-243b-53c5d68004f1
md"""
### Task \#18
"""

# ╔═╡ 9a4b6df6-67c3-11eb-27d6-2546dd023fb4
begin
	K = [
		1 2
		2 5
	]
	lu(K)
end

# ╔═╡ Cell order:
# ╠═ec794b0c-67b1-11eb-23e7-459641028308
# ╟─47f1758c-67b1-11eb-0a89-27d2595e5912
# ╠═9aa05ff6-67b5-11eb-0b36-95986372b105
# ╠═4533fdcc-67b3-11eb-30f8-851ef7fc43c9
# ╟─bf65578a-67b8-11eb-2627-9139089fcf68
# ╠═1c2ed1ac-67b5-11eb-1e77-67e96dbd2802
# ╟─1568a204-67b9-11eb-3d55-d72928803e5e
# ╠═ed7d86f6-67b3-11eb-273c-43770e6f299c
# ╟─c3dc9efe-67bd-11eb-0224-af9d70653b73
# ╠═e21e095c-67bd-11eb-12e4-e92b18b0b223
# ╟─67060602-67c2-11eb-046e-dfc437a3dc81
# ╠═832167be-67c2-11eb-1693-ab8ea1a74995
# ╟─8d7f475a-67c3-11eb-243b-53c5d68004f1
# ╠═9a4b6df6-67c3-11eb-27d6-2546dd023fb4


using GAFramework
using GAFramework.CoordinateGA
using LinearAlgebra
using FileIO
using Random

# Solves for function a + 2b + 3c + 4d + 5e = 42
# by minimizing abs(a + 2b + 3c + 4d + 5e - 42)
m = FunctionModel(x -> abs(x[1] + 2x[2] + 3x[3] + 4x[4] + 5x[5] - 42),
    [-100.,-100.,-100.,-100.,-100], # minimum corner
    [100.,100,100.,100.,100]) # maximum corner in rectangle
# Prints fitness every iteration
state = GAState(m, ngen=100, npop=3_000, elite_fraction=0.01,
    params=Dict(:mutation_rate=>0.1, :print_fitness_iter=>10))
println(ga!(state))

# Solves for function a + 2b + 3c + 4d + 5e = 42
# by minimizing abs(a + 2b + 3c + 4d + 5e - 42)
# without restricting the result to the [-100,100] rectangle
# but still initializing the population in that rectangle
m = FunctionModel(x -> abs(x[1] + 2x[2] + 3x[3] + 4x[4] + 5x[5] - 42),
    [-100.,-100.,-100.,-100.,-100], # minimum corner
    [100.,100,100.,100.,100], false) # maximum corner in rectangle
# Prints fitness every iteration
state = GAState(m, ngen=100, npop=3_000, elite_fraction=0.01,
    params=Dict(:mutation_rate=>0.1, :print_fitness_iter=>10))
println(ga!(state))

# Minimizes exp(|x|)
m = FunctionModel(x -> exp(sqrt(dot(x,x))),
    [-1.,-1.,-1.,-1.,-1], # minimum corner
    [1.,1.,1.,1.,1]) # maximum corner in rectangle
# Saves creature to file every 10 iterations
state = GAState(m, ngen=100, npop=3_000, elite_fraction=0.01,
    params=Dict(:mutation_rate=>0.1, :print_fitness_iter=>10,
        :save_creature_iter=>10, :file_name_prefix=>"minexp_3000"))
println(ga!(state))

# Change maximum number of generations and then continue
state.ngen = 200
ga!(state)

# Minimizes |tan(x + 1000)|
m = FunctionModel(x -> abs(tanh(x[1]+1000)), [-2000.], [2000.])
# Saves full state to file every 10 iterations
state = GAState(m, ngen=100, npop=3_000, elite_fraction=0.01,
    params=Dict(:mutation_rate=>0.1, :print_fitness_iter=>10,
        :save_state_iter=>25, :file_name_prefix=>"tan1000_3000"))
println(ga!(state))

if false
    # Reload state from file and restart
    state = loadgastate("tan1000_3000_state_25.jld")

    ga!(state)
end
# It's possible to also change the other fields in state and then continue on
# but that is usually only when you made a mistake with a filename or something


using Base.Test
using JLab
import PyPlot.plt

# #############################################################################
#Testing "getfit"
println("########## Testing \"getfit\" ##########")

#Sample data
x = linspace(0,10,100)
y = 1 + 3*x
y0 = y + 0.1*randn(100)
y0err = fill(0.1,length(y))
#y1 = y + (0.01*x+0.1).*randn(100)
y1 = [0.960833039028134,1.407268029451956,1.7041362562849804,1.7439408410837736,2.408365436088803,2.3558604670692187,2.6911766100562473,2.985346937471822,3.4325552973746216,3.6518536870040856,3.8198571085348556,4.249638029398669,4.5191875756320465,4.869643189063813,5.200098855165331,5.6028720774344825,5.8669985679112155,6.13446639319173,6.6589983913598445,6.583599635749586,7.0643398456691475,7.22881263766044,7.766208155795485,7.9279248867527095,8.031876149276476,8.622312209495371,8.91270431877012,9.253120846258275,9.46051424213136,9.864708345572803,10.196800366860534,10.262262734504358,10.703336679844554,11.159367909410706,11.448384037553025,11.419407340631926,11.882087020095623,12.123486147998683,12.589137110943783,12.915894183324713,13.195867464426314,13.381066468422823,13.498128075316496,13.946806258004903,14.140720490641081,14.65645590142797,14.92980075431186,15.422243674966104,15.606744487089607,15.664750441796148,16.035908118733797,16.4953299199874,16.958066212026043,16.994285363697312,17.354783816298067,17.700581921115532,18.0306766566351,18.382659996933242,18.231343132691336,19.063032842217314,19.255377262197005,19.81437282587234,19.956484257580982,19.97657224631401,20.360730031763094,20.65882754494313,21.106841540004787,21.252496222358445,21.766760201336467,22.19490679290051,22.6047978108247,22.529362909631008,22.915135996360842,23.17387598157397,23.43698104498548,23.598722726534746,23.73428978953588,24.241320075552377,24.65076910198924,24.78716650508645,25.05422998129448,25.58384488593317,25.93224573773362,25.843980681757863,26.45408533700855,26.888994042344866,27.25415441864504,27.83797148019114,27.550899599039617,28.03717936818957,28.18093241086242,28.548423792990217,28.591866793393848,29.208822396144086,29.49443477158012,29.793649140093873,29.867114283987977,30.526713526135133,30.7660738194201,30.89530630541283]
y1err = 0.01*x + 0.1

#Model
f(x,p) = p[1] + p[2]*x

#Get fit
fit = getfit(f,x,y,ones(Float64,length(y)),[0.,0.])
fit0 = getfit(f,x,y0,y0err,[0.,0.])
fit1 = getfit(f,x,y1,y1err,[0.,0.])

#Fit fields
println("Testing JLabFitresult fields ...")
@test fit0.xdata == x
@test fit0.ydata == y0
@test fit0.yerrors == y0err
@test fit0.model == f
@test fit0.dof == 98

#Fit accuracy
println("Testing accuracy ...")
@test_approx_eq_eps(fit.params[1],1,1e-10)
@test_approx_eq_eps(fit.params[2],3,1e-10)

p0 = sgolaycoeff(x,1)*y0
@test_approx_eq_eps(fit0.params[1],p0[1],1e-7)
@test_approx_eq_eps(fit0.params[2],p0[2],1e-10)

@test_approx_eq_eps(fit1.params[1],1,fit1.perrors[1])
@test_approx_eq_eps(fit1.params[2],3,fit1.perrors[2])

println("Function \"getfit\" passes tests.")


# # Bounds in Probability

#md # [![](https://mybinder.org/badge_logo.svg)](@__BINDER_ROOT_URL__/generated/Other Applications/bounds_in_probability.ipynb)
#md # [![](https://img.shields.io/badge/show-nbviewer-579ACA.svg)](@__NBVIEWER_ROOT_URL__/generated/Other Applications/bounds_in_probability.ipynb)
# **Adapted from**: SOSTOOLS' SOSDEMO8 (See Section 4.8 of [SOSTOOLS User's Manual](http://sysos.eng.ox.ac.uk/sostools/sostools.pdf))

using Test #src

# The probability adds up to one.

μ0 = 1

# The mean is one.

μ1  = 1

# The standard deviation is 1/2.

σ = 1/2

# The second moment `E(x^2)` is:

μ2 = σ^2 + μ1^2

# We define the moments as follows:

using DynamicPolynomials
@polyvar x
monos = [1, x, x^2]
using SumOfSquares
μ = measure([μ0, μ1, μ2], monos)

# We need to pick an SDP solver, see [here](https://jump.dev/JuMP.jl/v0.21.6/installation/#Supported-solvers) for a list of the available choices.
# We use `SOSModel` instead of `Model` to be able to use the `>=` syntax for Sum-of-Squares constraints.

using CSDP
solver = optimizer_with_attributes(CSDP.Optimizer, MOI.Silent() => true)
model = SOSModel(solver);

# We create a polynomial with the monomials in `monos` and
# JuMP decision variables as coefficients as follows:

@variable(model, poly, Poly(monos))

# Nonnegative on the support:

K = @set 0 <= x && x <= 5
con_ref = @constraint(model, poly >= 0, domain = K)

# Greater than one on the event:

@constraint(model, poly >= 1, domain = (@set 4 <= x && x <= 5))

# The bound (we use `LinearAlgebra` for the `⋅` syntax for the scalar product):

using LinearAlgebra
@objective(model, Min, poly ⋅ μ)

# We verify that we found a feasible solution:

optimize!(model)
@test primal_status(model) == MOI.FEASIBLE_POINT #src
primal_status(model)

# The objective value is `1/37`:

@test isapprox(objective_value(model), 1/37, rtol=1e-5) #src
objective_value(model)

# The solution is `(12x-11)^2 / 37^2`:

@test sos_decomposition(con_ref, K, 1e-4) isa SOSDecompositionWithDomain #src
@test isapprox(value(poly), ((12/37)x-11/37)^2, rtol=1e-3) #src
value(poly) * 37^2


# Test for gaussjacobi().
    tol = 1e-14
    # Test a small n:
    n = 42; a = -.1; b = .3
    x = gaussjacobi(n, -.1, .3)
    w = x[2]; x = x[1]
    @test all(length(x) == n) && all(length(w) == n)
    @test abs(x[37] - 0.912883347814032) < tol
    @test abs(w[37] - 0.046661910947553) < tol

    # Test a larger n (using ASY)
    a = -.7; b = 1.3;
    n = 251;
    x = gaussjacobi(n,-.7,1.3);
    w = x[2]; x = x[1]
    @test all(length(x) == n) && all(length(w) == n)
    @test abs(x[37] + 0.893103435898983) < tol
    @test abs(w[37] - 1.962534523788093e-04) < tol

    # Test n = 1:
    a = 1.0; b = 2.0;
    x = gaussjacobi( 1, a, b );
    @test abs( x[1][1] - (b-a)/(a+b+2) ) < tol
    @test abs( x[2][1] - 2^(a+b+1)*beta(a+1, b+1) ) < tol

    x = gaussjacobi( 1013, .9, -.1 )
    @test abs( x[1][2]  + 0.999986012231899 ) < tol
    @test abs( x[1][13]  + 0.999225722939832 ) < tol
    @test abs( x[2][2] - 9.314674169892358e-05 ) < tol
    @test abs( x[2][13] - 4.654651764553262e-04 ) < tol

    x = gaussjacobi( 10013, .9, -.1 )
    @test abs( x[1][2]  +0.999999856605293 ) < tol
    @test abs( x[1][13]  + 0.999992061552711 ) < tol
    @test abs( x[2][2] - 1.509654630405615e-06 ) < tol
    @test abs( x[2][13] - 7.548275262993863e-06 ) < tol


using StatsBase
using Statistics
using Test

function learn_MeanFunctions()
    data = [1:5...]

    @test 3.0 == mean(data)
end

function learn_ScalarStatistics()
    data = [1:5...]
    count = length(data)

    # var
    @test 2.0 == var(data, corrected=false) # population: /n from Statistics
    @test 2.0 == var(data, uweights(count)) # population: /n from StatsBase

    @test 2.5 == var(data)                                  # sample: /(n-1) from Statistics
    @test 2.5 == var(data, uweights(count), corrected=true) # sample: /(n-1) from StatsBase

    # std
    @test sqrt(2.0) == std(data, corrected=false) # population: /n from Statistics
    @test sqrt(2.0) == std(data, uweights(count)) # population: /n from Statistics

    @test sqrt(2.5) == std(data)
    @test sqrt(2.5) == std(data, uweights(count), corrected=true)

    # zscore = (data - mu)/sigma
    mu = 2.0          # mean, from above
    sigma = sqrt(2.0) # std, from above
    @test zscore(data, mu, sigma) == [-0.7071067811865475, 0.0, 0.7071067811865475, 1.414213562373095, 2.1213203435596424]
end

@testset "StatsBase" begin
    learn_MeanFunctions()
    learn_ScalarStatistics()
end

nothing


# plots the contours of the OLS objective function,
# with the true and estimated coefficients
# generate data
using Plots, Econometrics
n = 20
x = [ones(n) randn(n)]
beta = randn(2)
y = x*beta+randn(n)
# ols estimate and objective function
betahat = x\y
s = (a,b)-> (1/n)*sum((y - x*[a,b]).^2)
# plot the contours and the points
closeall()
b1 = range(-4.0,stop=4.0,length=100)
b2 = range(-4.0,stop=4.0,length=100)
p = contour(b1,b2,(b1,b2)->s(b1,b2),fill=true, c=:viridis)
scatter!([beta[1]],[beta[2]], markersize=10, label="true")
scatter!([betahat[1]],[betahat[2]], markersize=10, label="estimated")
# do OLS by simulated annealing, and plot the path
lb = -4*ones(2,1)
ub = -lb
thetastart = fill(-3.0, 2, 1)
obj = theta-> (1/n)*sum((y - x*theta).^2)[1,1]
results = samin(obj, thetastart, lb, ub, verbosity=2)
path = results[4]
plot!(path[:,4], path[:,5])


using Plots; pyplot()
using DeSolver
using LinearAlgebra
using LaTeXStrings

# Number of collocation points
N = 50
h= pi/N

# create function
function f(u,t)
    du = zeros(length(u))
    for i in 1:length(u)
        if i ==1
            du[1] = (-2u[1] + 2u[2])/h^2
        elseif i == length(u)
            du[length(u)] = (2u[length(u)-1] - 2u[length(u)])/h^2
        else
            du[i] = (u[i-1] - 2u[i] + u[i+1])/h^2
        end
    end
    return du
end

xs=range(0, stop=pi, length=N)
u0=sin.(xs)

soltest1=odesolve(f, u0, (0.,10.), 1, method="RK4")
soltest2=odesolve(f, u0, (0.,10.), .1, method="RK4")
soltest3=odesolve(f, u0, (0.,10.), .01, method="RK4")


# By default, Julia/LLVM does not use fused multiply-add operations (FMAs).
# Since these FMAs can increase the performance of many numerical algorithms,
# we need to opt-in explicitly.
# See https://ranocha.de/blog/Optimizing_EC_Trixi for further details.
@muladd begin

"""
    ln_mean(x, y)

Compute the logarithmic mean

    ln_mean(x, y) = (y - x) / (log(y) - log(x)) = (y - x) / log(y / x)

Problem: The formula above has a removable singularity at `x == y`. Thus,
some care must be taken to implement it correctly without problems or loss
of accuracy when `x ≈ y`. Here, we use the approach proposed by
Ismail and Roe (2009).
Set ξ = y / x. Then, we have

    (y - x) / log(y / x) = (x + y) / log(ξ) * (ξ - 1) / (ξ + 1)

Set f = (ξ - 1) / (ξ + 1) = (y - x) / (x + y). Then, we use the expansion

    log(ξ) = 2 * f * (1 + f^2 / 3 + f^4 / 5 + f^6 / 7) + O(ξ^9)

Inserting the first few terms of this expansion yields

    (y - x) / log(ξ) ≈ (x + y) * f / (2 * f * (1 + f^2 / 3 + f^4 / 5 + f^6 / 7))
                     = (x + y) / (2 + 2/3 * f^2 + 2/5 * f^4 + 2/7 * f^6)

Since divisions are usually more expensive on modern hardware than
multiplications (Agner Fog), we try to avoid computing two divisions. Thus,
we use

    f^2 = (y - x)^2 / (x + y)^2
        = (x * (x - 2 * y) + y * y) / (x * (x + 2 * y) + y * y)

Given ε = 1.0e-4, we use the following algorithm.

    if f^2 < ε
      # use the expansion above
    else
      # use the direct formula (y - x) / log(y / x)
    end

# References
- Ismail, Roe (2009).
  Affordable, entropy-consistent Euler flux functions II: Entropy production at shocks.
  [DOI: 10.1016/j.jcp.2009.04.021](https://doi.org/10.1016/j.jcp.2009.04.021)
- Agner Fog.
  Lists of instruction latencies, throughputs and micro-operation breakdowns
  for Intel, AMD, and VIA CPUs.
  https://www.agner.org/optimize/instruction_tables.pdf
"""
@inline function ln_mean(x, y)
  epsilon_f2 = 1.0e-4
  f2 = (x * (x - 2 * y) + y * y) / (x * (x + 2 * y) + y * y) # f2 = f^2
  if f2 < epsilon_f2
    return (x + y) / @evalpoly(f2, 2, 2/3, 2/5, 2/7)
  else
    return (y - x) / log(y / x)
  end
end

"""
    inv_ln_mean(x, y)

Compute the inverse `1 / ln_mean(x, y)` of the logarithmic mean
[`ln_mean`](@ref).

This function may be used to increase performance where the inverse of the
logarithmic mean is needed, by replacing a (slow) division by a (fast)
multiplication.
"""
@inline function inv_ln_mean(x, y)
  epsilon_f2 = 1.0e-4
  f2 = (x * (x - 2 * y) + y * y) / (x * (x + 2 * y) + y * y) # f2 = f^2
  if f2 < epsilon_f2
    return @evalpoly(f2, 2, 2/3, 2/5, 2/7) / (x + y)
  else
    return log(y / x) / (y - x)
  end
end

# `Base.max` and `Base.min` perform additional checks for signed zeros and `NaN`s
# which are not present in comparable functions in Fortran/C++. For example,
# ```julia
# julia> @code_native debuginfo=:none syntax=:intel max(1.0, 2.0)
#         .text
#         vmovq   rcx, xmm1
#         vmovq   rax, xmm0
#         vcmpordsd       xmm2, xmm0, xmm0
#         vblendvpd       xmm2, xmm0, xmm1, xmm2
#         vcmpordsd       xmm3, xmm1, xmm1
#         vblendvpd       xmm3, xmm1, xmm0, xmm3
#         vmovapd xmm4, xmm2
#         test    rcx, rcx
#         jns     L45
#         vmovapd xmm4, xmm3
# L45:
#         test    rax, rax
#         js      L54
#         vmovapd xmm4, xmm3
# L54:
#         vcmpltsd        xmm0, xmm0, xmm1
#         vblendvpd       xmm0, xmm4, xmm2, xmm0
#         ret
#         nop     word ptr cs:[rax + rax]
#
# julia> @code_native debuginfo=:none syntax=:intel min(1.0, 2.0)
#         .text
#         vmovq   rcx, xmm1
#         vmovq   rax, xmm0
#         vcmpordsd       xmm2, xmm0, xmm0
#         vblendvpd       xmm2, xmm0, xmm1, xmm2
#         vcmpordsd       xmm3, xmm1, xmm1
#         vblendvpd       xmm3, xmm1, xmm0, xmm3
#         vmovapd xmm4, xmm2
#         test    rcx, rcx
#         jns     L58
#         test    rax, rax
#         js      L67
# L46:
#         vcmpltsd        xmm0, xmm1, xmm0
#         vblendvpd       xmm0, xmm4, xmm2, xmm0
#         ret
# L58:
#         vmovapd xmm4, xmm3
#         test    rax, rax
#         jns     L46
# L67:
#         vmovapd xmm4, xmm3
#         vcmpltsd        xmm0, xmm1, xmm0
#         vblendvpd       xmm0, xmm4, xmm2, xmm0
#         ret
#         nop     word ptr cs:[rax + rax]
#         nop     dword ptr [rax]
# ```
# In contrast, we get the much simpler and faster version
# ```julia
# julia> @code_native debuginfo=:none syntax=:intel Base.FastMath.max_fast(1.0, 2.0)
#         .text
#         vmaxsd  xmm0, xmm1, xmm0
#         ret
#         nop     word ptr cs:[rax + rax]
#
# julia> @code_native debuginfo=:none syntax=:intel Base.FastMath.min_fast(1.0, 2.0)
#         .text
#         vminsd  xmm0, xmm0, xmm1
#         ret
#         nop     word ptr cs:[rax + rax]
# ```
# when using `@fastmath`, which we also get from
# [Fortran](https://godbolt.org/z/Yrsa1js7P)
# or [C++](https://godbolt.org/z/674G7Pccv).
"""
    max(x, y, ...)

Return the maximum of the arguments. See also the `maximum` function to take
the maximum element from a collection.

This version in Trixi.jl is semantically equivalent to `Base.max` but may
be implemented differently. In particular, it may avoid potentially expensive
checks necessary in the presence of `NaN`s (or signed zeros).

# Examples

julia> max(2, 5, 1)
5
"""
@inline max(args...) = @fastmath max(args...)

"""
    min(x, y, ...)

Return the minimum of the arguments. See also the `minimum` function to take
the minimum element from a collection.

This version in Trixi.jl is semantically equivalent to `Base.min` but may
be implemented differently. In particular, it may avoid potentially expensive
checks necessary in the presence of `NaN`s (or signed zeros).

# Examples

julia> min(2, 5, 1)
1
"""
@inline min(args...) = @fastmath min(args...)

end # @muladd


#Algorithm 12.2
#Heat equation with backwards-difference
#Boundary conditions u(0, t) = u(l, t) = 0, 0 < t < T

using Printf

function heatbd(f, L, T, α, m, N)
	w = zeros(Float64, m)
	l = zeros(Float64, m)
	u = zeros(Float64, m)
	z = zeros(Float64, m)
	h = L/m
	k = T/N
	λ = k*α^2/h^2
	for i ∈ 1:(m - 1)
		w[i] = f(i * h)
	end
	l[1] = 1 + 2λ
	u[1] = -λ/l[1]
	for i ∈ 2:(m - 2)
		l[i] = 1 + 2λ + λ*u[i - 1]
		u[i] = -λ/l[i]
	end
	l[m - 1] = 1 + 2λ + λ*u[m - 2]
	for j ∈ 1:N
		t = j*k
		z[1] = w[1] / l[1]
		for i ∈ 2:(m - 1)
			z[i] = (w[i] + λ*z[i - 1])/l[i]
		end
		w[m - 1] = z[m - 1]
		for i ∈ (m - 2):1
			w[i] = z[i][t] - u[i]w[i+1]
		end
		for i ∈ 1:(m - 1)
			x = i*h
			println(x, w[i])
			x = i * h
			@printf("%3d %11.8f %14.8f\n", i, x, w[i])
		end
	end
end

function distrib(x)
	return sin(π*x)
end

heatbd(distrib, 1, 10, 0.5, 100, 100)


# plot probabilistic distribution as bar graph
# using cumulative distribution function

module BarProbDistCdf
    using Plots, Distributions
    pyplot()

    function main(is_test=false)
        norm_dist = Normal(209.7, 4.84) # mu, std_dev

        zs = Array(190:230)
        ys = [cdf(norm_dist, z+0.5) - cdf(norm_dist, z-0.5) for z in zs]

        bar(ys, xtick=zs, label="Prob Dist")

        if is_test == false
            save_path = joinpath(split(@__FILE__, "src")[1], "src/prob_stats/prob_model/bar_prob_dist_cdf/bar_prob_dist_cdf.png")
            savefig(save_path)
        end
    end
end


# ---
# title: 172. Factorial Trailing Zeroes
# id: problem172
# author: zhwang
# date: 2022-03-25
# difficulty: Easy
# categories: Math
# link: <https://leetcode.com/problems/factorial-trailing-zeroes/description/>
# hidden: true
# ---
#
# Given an integer `n`, return _the number of trailing zeroes in`n!`_.
#
# **Follow up:** Could you write a solution that works in logarithmic time
# complexity?
#
#
#
# **Example 1:**
#
#
#
#     Input: n = 3
#     Output: 0
#     Explanation:  3! = 6, no trailing zero.
#
#
# **Example 2:**
#
#
#
#     Input: n = 5
#     Output: 1
#     Explanation:  5! = 120, one trailing zero.
#
#
# **Example 3:**
#
#
#
#     Input: n = 0
#     Output: 0
#
#
#
#
# **Constraints:**
#
#   * `1 <= n <= 104`
#
#
## @lc code=start
using LeetCode

## simulate Mathematica's function NestWhileList
function nest_while_list(f::Function, val::T, chk::Function)::Vector{T} where T
    res = [val]
    while chk(val)
        val = f(val)
        push!(res, val)
    end
    res
end

trailing_zeroes(n::Int) = sum(nest_while_list(i->i÷5, n÷5, >(1)))
## equivalent to the following
## function trailing_zeroes(n::Int)::Int
##     res = 0
##     while n >=5
##         n ÷= 5
##         res += n
##     end
##     res
## end

## @lc code=end

## @lc test=start
@testset "172.factorial-trailing-zeroes.jl" begin
    @test all(trailing_zeroes(i) == 0 for i in 0:4)
    @test trailing_zeroes(5) == 1
    @test trailing_zeroes(1999) == 496
    @test trailing_zeroes(2000) == trailing_zeroes(2001) == 499
end
## @lc test=end

## @lc info=start
# link: [solution to 172]()
## @lc info=end


# Beta distribution

export Beta

@parameterized Beta(α,β)

@kwstruct Beta(α, β)

@kwalias Beta [
    a     => α
    alpha => α
    b     => β
    beta  => β
]

TV.as(::Beta) = as𝕀

function logdensity(d::Beta{(:α, :β), Tuple{A,B}}, x::X) where {A,B,X}
    if static_hasmethod(xlogy, Tuple{A,X}) && static_hasmethod(xlog1py, Tuple{B,X})
        return xlogy(d.α - 1, x) + xlog1py(d.β - 1, -x)
    else
        return (d.α - 1) * log(x) + (d.β - 1) * log1p(-x)
    end
end

function basemeasure(d::Beta{(:α,:β)})
    inbounds(x) = 0 < x < 1
    constℓ = 0.0
    varℓ() = - logbeta(d.α, d.β)
    base = Lebesgue(ℝ)
    FactoredBase(inbounds, constℓ, varℓ, base)
end

Base.rand(rng::AbstractRNG, T::Type, μ::Beta) = rand(rng, Dists.Beta(μ.α, μ.β))

distproxy(d::Beta{(:α, :β)}) = Dists.Beta(d.α, d.β)

asparams(::Type{<:Beta}, ::Val{:α}) = asℝ₊
asparams(::Type{<:Beta}, ::Val{:β}) = asℝ₊


"""
    pinv_M, rnull_optional, lnull_optional = pinv_qr(M::AbstractArray)

Computes Moore-Penrose pseudoinverse of `M` using a rank-revealing QR
factorization and optionally returns an orthonormal basis for right
and left null-space. This can be significantly faster than using
SVD for computing the pseudoinverse.  Moreover, `A` can be sparse or
dense.

# Examples
```julia-repl
julia> pinv_qr([3 8 17; 2 1 4; 8 3 21])
([-0.10344827586206914 1.344827586206895 -0.17241379310344793;0.1149425287356321 0.8390804597701144 -0.2528735632183907;0.022988505747126506 -0.6321839080459762 0.1494252873563217], Matrix{Float64}(undef, 3, 0), Matrix{Float64}(undef, 3, 0))
```
"""
function pinv_qr(M::AbstractArray{T}) where {T}
    m = size(M, 1)
    n = size(M, 2)

    F = qr(M)
    Q = F.Q*I  # so Q is allways m x m, with Matrix(F.Q) we get a m x n Matrix for m >= n
    R = Matrix(F.R)

    # compute rank
    abs_diag_R = abs.(diag(R))
    tol = 100 * minimum(size(M)) * maximum(abs_diag_R) * eps(T)  # MAGIC CONSTANT
    rnk = sum(tol .< abs_diag_R)

    # compute pseudoinverse
    M_pinv = Matrix{T}(undef, n, m)

    S = R[1:rnk, 1:rnk] \ R[1:rnk, rnk+1:end]
    W = R[1:rnk, 1:rnk] \ Q[:, 1:rnk]'
    X = (S' * S + I) \ (S' * W)
    M_pinv[rnk+1:end, :] = X
    M_pinv[1:rnk, :] = W - S*X

    # computing null spaces
    right_null_size = n - rnk
    left_null_size = m - rnk

    # right null space
    if right_null_size > 0
        rnull, = qr(Matrix([S; -I(right_null_size)]))
        rnull = Matrix(rnull)
    else
        rnull = zeros(T, n, 0)
    end

    # left null space
    if left_null_size > 0
        lnull = Q[: ,end - (left_null_size - 1): end]
    else
        lnull = zeros(T, m, 0)
    end

    return M_pinv, rnull, lnull
end

function pinv_qr(M::AbstractArray{<:Int})
    pinv_qr(float(M))
end


using ImplicitGraphs, Permutations

"""
    iTransposition(n::Int, adjacent::Bool=true)

Create an `ImplicitGraph` whose vertices are all permutations of `1:n`
in which two vertices are adjacent if they differ by a transpostion.

* If `adjacent` is true, then only consider transpositions of the form `(a,a+1)`
* Otherwise, consider all possible transpositions of the form `(a,b)` where `1≤a<b≤n`.
"""
function iTransposition(n::Int, adjacent::Bool = true)
    @assert n > 0 "Require n>0, got n=$n"

    vcheck(v::Permutation) = length(v) == n

    function outs(v::Permutation)
        if adjacent
            return [v * Transposition(n, i, i + 1) for i = 1:n-1]
        end
        return [v * Transposition(n, i, j) for i = 1:n for j = 1:n if i ≠ j]
    end

    return ImplicitGraph{Permutation}(vcheck, outs)
end

"""
    trans_string(p::Permutation)

We assume that `p` is a transposition.
"""
function trans_string(p::Permutation)::String
    n = length(p)
    FP = fixed_points(p)
    @assert n == length(FP) + 2 "Permutation must be a transposition"
    elts = sort(setdiff(1:n, FP))
    a, b = elts
    return "($a,$b)"
end

"""
    pscore(p::Permutation)::Int

Measures the extent to which `p` is different from the identity permutation
using this formula:

`sum(abs(p[k]-k) for k=1:length(p))`
"""
function pscore(p::Permutation)::Int
    n = length(p)
    sum(abs(i - p(i)) for i = 1:n)
end

"""
    crazy_trans_product(p::Permutation, adjacent::Bool=true)::String

Express `p` as the product of transpositions. With `adjacent=true` the transpositions
are all of the form `(a,a+1)`; otherwise, all possible transpositions are allowed.

Result is expressed as a `String`.
"""
function crazy_trans_product(p::Permutation, adjacent::Bool = true)::String
    n = length(p)
    G = iTransposition(n, adjacent)
    P = reverse(guided_path_finder(G, p, Permutation(n), score = pscore))

    nP = length(P)

    if nP == 1
        return "()"
    end

    Q = [P[k]' * P[k+1] for k = 1:nP-1]

    prod(trans_string(Q[j]) for j = 1:length(Q))
end


logit(v::Float64) = log(v/(1-v))

inv_logit(w::Float64) = exp(w)/(1+exp(w))


using Test
using ManifoldProjections
using LinearAlgebra
using Random
Random.seed!(0)

# test sphere with radius 1
x = randn(4)+im*randn(4)
M = Sphere()
retract!(M,x)
@test norm(x) ≈ 1
v = randn(4)+im*randn(4)
@test project_tangent(M,v,x) == project_tangent!(M,v,x)
@test abs(real(v'*x)) < 1e-8
ε = 1e-8
@test (norm(x + ε*v) - 1) < ε^(3/2) # the tangent vector is such that x+εv is on the manifold up to O(ε^2)

# test sphere with radius r
r = 120.0
x = randn(4)+im*randn(4)
M = Sphere(r)
retract!(M,x)
@test norm(x) ≈ r
v = randn(4)+im*randn(4)
@test project_tangent(M,v,x) == project_tangent!(M,v,x)
@test abs(real(v'*x)) < 1e-8
@test (norm(x + ε*v) - r) < ε^(3/2) # the tangent vector is such that x+εv is on the manifold up to O(ε^2)

for M in (Stiefel_CholQR(), Stiefel_SVD())
    X = randn(4,2) + im*randn(4,2)
    retract!(M,X)
    @test X'X ≈ I
    V = randn(4,2)+im*randn(4,2)
    @test project_tangent(M,V,X) == project_tangent!(M,V,X)
    @test norm((X + ε*V)'*(X+ε*V) - I) < ε^(3/2)
end


### X9.62 spec talks about the way optimal basis is selected as for polynomial and for gaussian normal basis. Currently, selection rule is implemented only for gaussian normal basis

function _isprime(p)

    for i in 2:(p - 1)
        mod(p, i) == 0 && return false
    end

    return true
end

function gn_basis_exist(m, T)

    @assert mod(m, 8) != 0 # May as well return false?

    p = T*m + 1

    !_isprime(p) && return false

    k = compute_integer_order(2, p)

    h = div(T*m, k)

    d = gcd(h, m)

    if d == 1
        return true
    else
        return false
    end

end

function gn_basis_representation_rule(m)

    @assert mod(m, 8) != 0

    if gn_basis_exist(m, 2)
        return 2
    elseif gn_basis_exist(m, 1)
        return 1
    else
        T = 3

        while true
            if gn_basis_exist(m, T)
                return T
            end
            T += 1
        end
    end
end

function specialize(::Type{F2GNB}, N::Int)

    @assert div(N, 8) != 0 "Out of X9.62 spec"
    T = gn_basis_representation_rule(m)

    return F2GNB{N, T}
end

function F2GNB(x::BitVector)

    N = length(x)
    F = specialize(F2GNB, N)

    return F(x)
end


#=
Each new term in the Fibonacci sequence is generated by adding the previous two terms. By starting with 1 and 2, the first 10 terms will be:

1, 2, 3, 5, 8, 13, 21, 34, 55, 89, ...

By considering the terms in the Fibonacci sequence whose values do not exceed four million, find the sum of the even-valued terms.
=#

function calc()
  fibs = [1, 1]
  while fibs[end] <= 4_000_000
    push!(fibs, fibs[end] + fibs[end-1])
  end
  filter(x -> iseven(x) && x < 4_000_000, fibs) |> sum |> println
end
@time calc()


# some utility functions
const Γ = gamma

function generalized_binomial(α::T, n::S) where {T,S <: Integer}
    R = Base.promote_op(/,T,S)
    out = one(R)
    for i in n:-1:1
        out *= α/i
        α -= 1
    end
    out
end

## memoized version of the above
@memoize function generalized_binomial′(a, k::Int)
    k < 0 && return NaN * one(a)
    k == 0 && return one(a)/1
    generalized_binomial′(a,k-1) * (a-k+1)/k
end

## x⁽ⁱ⁾ is falling (x)⋅(x-1)⋯(x-n+1)
function Pochhammer(::Val{:falling}, z::T,n::Int) where {T}
    iszero(n) && return one(T)
    out = z
    for i in 1:n-1
        z -= 1
        out *= z
    end
    out
end

## (x)ᵢ is rising (x)⋅(x+1)⋯(x+n-1) = Γ(z+n)/Γ(z)
function Pochhammer(::Val{:rising}, z::T,n::Int) where {T}
    iszero(n) && return one(T)
    out = z
    for i in 1:n-1
        z += 1
        out *= z
    end
    out
end
# default is (x)ᵢ
Pochhammer(z,n::Int) = Pochhammer(Val(:rising),  z, n)

# (x)_n/n!
Pochhammer_factorial(z, n) = Pochhammer_factorial(Val(:rising), z, n)

function Pochhammer_factorial(::Val{:rising}, z, n)
    iszero(n) && return one(z)/1
    prod((z+i)/(n-i) for i in 0:n-1)
end

## Alternative to HypergeometricFunctions.jl so that
## variables can be inserted into the numerator
"""
    pFq(as, bs, z; [maxevals=1000])

Compute the generalized [hypergeometric](https://en.wikipedia.org/wiki/Generalized_hypergeometric_function) function. The `HypergeometricFunction.jl` package would normally be used for such calculations, but this one accepts polynomial values for the `as` and `z` values.

# Example

From [mathworld](https://mathworld.wolfram.com/HypergeometricFunction.html)

```jldoctest
julia> using Polynomials, SpecialPolynomials

julia> import SpecialPolynomials: pFq, Pochhammer

julia> pFq((1/3,2/3), 5/6, 27/32) ≈ 8/5
true

julia> pFq([1/4, 1/2], [3/4], 80/81; maxevals=2000) ≈ 9/5
true

julia> x = variable()
Polynomial(x)

julia> n = 5
5

julia> pFq((-n,n+1), 1, (1-x)/2) ≈ basis(Legendre,n)(x)
true

julia> α, β, n = 1/2, 1/2, 5;

julia> Pochhammer(α+1,n)/factorial(n) * pFq((-n,n+α+β+1), α+1, (1-x)/2) ≈ basis(Jacobi{α,β}, n)(x)
true
```
"""
function pFq(as, bs, z; maxevals=1000)
    n = 1
    acc = trm = one(z)

    p,q = length(as), length(bs)

    # element in `as` is negative integer or 0 && converges
    # p ≤ q && converges
    # p = q + 1 |x| < 1 && converge
    # p > q + 1 && diverges
    while n < maxevals
        a = isempty(as) ? 1 : prod(as)
        b = isempty(bs) ? 1 : prod(bs)

        iszero(a) && return acc
        iszero(b) && return Inf  # check on b

        trm *= a /b * z  /n
        acc += trm

        as = plus_1(as)

        bs = plus_1(bs)
        n += 1
    end

    if (p > q + 1 || (p == q+1 && abs(z) < 1))
        return acc
    else
        NaN*acc
    end
end
## tuples
@inline plus_1(as) = map(x->x+1, as)

# use HypergeometricFunction.mFn if possible
function pFq(as::Tuple{A,B}, bs::Tuple{C}, z::AbstractFloat) where {A,B,C}
    HypergeometricFunctions._₂F₁(as[1],as[2],bs[1],z)
end
function pFq(as::Tuple{A,B,C}, bs::Tuple{D,E}, z::AbstractFloat) where {A,B,C,D,E}
    HypergeometricFunctions._₃F₂(as[1],as[2],as[3],bs[1],bs[2],z)
end

## Try to speed up  quadgk by not specializing on F
mutable struct Wrapper
    F
end
(F::Wrapper)(x) = F.F(x)

_quadgk(f, a, b) = quadgk(Wrapper(f), a, b)[1]
const ∫ = _quadgk

checked_div(a, b) = (iszero(a) && iszero(b)) ? a : a/b


#=
  Cookie problem.

  From Think Bayes, page 3
  """
  Suppose there are two bowls of cookies.
  Bowl 1 contains 30 vanilla cookies and 10 chocolate cookies.
  Bowl 2 contains 20 of each.

  Now suppose you choose one of the bowls at random and, without looking,
  select a cookie at random. The cookie is vanilla.
  What is the probability that it came from Bowl 1?
  """

  Distributions of variable bowl (num:0)
  1.00000 =>    5991  (0.599100)
  2.00000 =>    4009  (0.400900)

  Cf  ~/webppl/cookie_problem.wppl

=#

using Turing, StatsPlots, DataFrames
include("jl_utils.jl")

@model function cookie_problem()
    vanilla = 1
    chocolate = 2

    bowl1 = 1
    bowl2 = 2

    bowl ~ DiscreteUniform(bowl1,bowl2)
    cookie ~ (bowl == bowl1) ? Categorical(simplex([30,10])) : # [vanilla,chocolate]
                               Categorical(simplex([20,20]))

    true ~ Dirac(cookie == vanilla)

end

model = cookie_problem()
num_chains = 4

# chns = sample(model, Prior(), 10_000)
# chns = sample(model, MH(), 10_000)
chns = sample(model, PG(5), 10_000)
# chns = sample(model, SMC(), 10_000)
# chns = sample(model, IS(), 10_000)

display(chns)
# display(plot(chns))

show_var_dist_pct(chns, :bowl)


using DifferentialEquations
using Plots
using Markdown
using LinearAlgebra
using StaticArrays

description=md"""

solve the matrix system
``u' = Au``
for matrix A and i.c. u₀

"""
display(description)

# use immutable arrays that are stack-allocated
A = @SMatrix [1.0 0 0 -5
              4 -2 4 -3
              -4 0 0 1
              5 -2 2 3]

u0 = rand(4, 2)

tspan = (0.0, 1.0)

f(du, u, p, t) = mul!(du, A, u)  # in-place version of matrix multiply

prob = ODEProblem(f, u0, tspan)
sol = solve(prob)
plot(sol)


export is_anagram, anagram_graph

"""
    is_anagram(w1,w2)
Checks if words `w1` and `w2` are anagrams of each other.

All letters are reduced to lower case so `Act` and `cat` are
considered anagrams.
"""
function is_anagram(w1::String, w2::String)::Bool
    c1 = counter(collect(lowercase(w1)))
    c2 = counter(collect(lowercase(w2)))
    return c1 == c2
end

function make_anagrams(word::String)::Vector{String}
    anags = unique(collect(permutations(word)))
    return join.(anags)
end

"""
    anagram_graph(S, len=0, trim=false)
Create a graph whose vertices are words in `S` of length `len` in which two words are
adjacent if they are anagrams of each other.  If `len` is zero, all words are used.
If `trim` is true, vertices of degree zero are removed.

Note that every component of this graph is a clique.
"""
function anagram_graph(
    S::Set{String},
    len::Int = 0,
    trim::Bool = false,
)::SimpleGraph{String}
    G = _bare_graph(S, len)

    # add edges

    VV = deepcopy(G.V)
    n = NV(G)
    PM = Progress(n)

    while length(VV) > 0
        next!(PM)
        v = first(VV)
        anas = [ana for ana ∈ make_anagrams(v) if ana ∈ G.V]
        nana = length(anas)
        for i = 1:nana-1
            for j = i+1:nana
                add!(G, anas[i], anas[j])
            end
            next!(PM)
        end
        for a in anas
            delete!(VV, a)
        end
    end

    # remove vertices of degree zero (if trim is true)
    if trim
        for w in G.V
            if deg(G, w) == 0
                delete!(G, w)
            end
        end
    end

    if len < 1
        name(G, "Anagram Graph on all words")
    else
        name(G, "Anagram Graph on $len-letter words")
    end

    return G

end


# -*- coding: utf-8 -*-
# ---
# jupyter:
#   jupytext:
#     formats: ipynb,jl:hydrogen
#     text_representation:
#       extension: .jl
#       format_name: hydrogen
#       format_version: '1.3'
#       jupytext_version: 1.11.2
#   kernelspec:
#     display_name: Julia 1.7.0-DEV
#     language: julia
#     name: julia-1.7
# ---

# %%
using Plots
plotly()
f(x, y) = sin(x) * sin(y) + (x - 2y)^2/50
x = range(-π, π; length=401)
y = range(-π, π; length=401)
surface(x, y, f; xlim=extrema(x), ylim=extrema(y), camera=(30, 45), size=(500, 500), colorbar=false)

# %%
fmin, pt = findmin(p -> f(p...), Iterators.product(x, y))

# %% tags=[]
surface(x, y, f; xlim=extrema(x), ylim=extrema(y), camera=(30, 45), size=(500, 500), colorbar=false)
scatter!([pt[1]], [pt[2]], [f(pt...)]; label="", color=:cyan, ms=2)

# %%
f(p) = sin(p[1]) * sin(p[2]) + (p[1] - 2p[2])^2/50
x = y = range(-π, π; length=401)
findmin(f, Iterators.product(x, y))

# %%
f(p) = sin(p[1]) * sin(p[2]) + (p[1] - 2p[2])^2/50
x = y = range(-π, π, 1000)
findmin(f, Iterators.product(x, y))

# %%
using Plots
plotly()
x = y = range(-3, 3, length=100)
f(x, y) = exp(-x^2-y^2)
z = f.(x', y)
surface(x, y, z; colorbar=false, size=(600, 600), color=:jet)

# %%
using Plots
plotly()
x = y = range(-3, 3, length=100)
z = exp.(.-x'.^ 2 .- y.^ 2)
surface(x, y, z; colorbar=false, size=(600, 600), color=:jet)

# %%
using Plots
plotly()
x = y = range(-3, 3, length=100)
z = @. exp(-x'^2 .- y^2)
surface(x, y, z; colorbar=false, size=(600, 600), color=:jet)

# %%

0004/v1.7 findmin and Plots plotly example.jl


#=

  Linear regression

  This is actually a WebPPL example (example/linear_regression.wppl) but translated via my BLOG model.

  xs = [0,1,2,3]
  ys = [0,1,4,6]

  mu should be ~ 2 and b ~ 0.
  y4: y for x=4 should be 2*4+0 = 8
  y5: y for x=5 should be 2*5+0 = 10

  Summary Statistics
  parameters      mean       std   naive_se      mcse         ess      rhat   ess_per_sec
      Symbol   Float64   Float64    Float64   Float64     Float64   Float64       Float64

           m    2.0134    0.3960     0.0040    0.0088   1980.8480    1.0001      445.5349
           b   -0.2678    0.7118     0.0071    0.0135   2785.0095    0.9999      626.4079
       sigma    0.8463    0.4970     0.0050    0.0155    929.0047    0.9999      208.9529
          y4    7.7715    1.5278     0.0153    0.0314   2240.9116    1.0001      504.0287
          y5    9.7916    1.8747     0.0187    0.0401   2060.8303    1.0000      463.5246

  With 5 data points (x=[0,1,2,3,4] and y=[0,1,4,6,8]) with got a much better result:
  Summary Statistics
  parameters      mean       std   naive_se      mcse         ess      rhat   ess_per_sec
      Symbol   Float64   Float64    Float64   Float64     Float64   Float64       Float64

           m    2.0000    0.0001     0.0000    0.0000    684.0973    0.9999      170.6830
           b    0.0000    0.0004     0.0000    0.0000   1140.3205    1.0005      284.5111
       sigma    0.0002    0.0003     0.0000    0.0000    111.5388    1.0486       27.8290
          y4    8.0000    0.0005     0.0000    0.0000    713.2871    1.0046      177.9658
          y5   10.0000    0.0006     0.0000    0.0000    553.0126    1.0016      137.9772

  cf ~/blog/linear_regression2.blog
     ~/webppl/linear_regression2.wppl
=#

using Turing
include("jl_utils.jl")

@model function linear_regression2(xs,ys)
    n = length(xs)

    m ~ Normal(0,2)
    b ~ Normal(0,2)
    sigma ~ Gamma(1,1)

    xs ~ filldist(Uniform(0,14),n)

    for i in 1:n
        ys[i] ~ Normal(m*xs[i] + b, sigma)
    end

    # Prediction for x=4 and x=5
    y4 ~ Normal(m*4 + b, sigma)
    y5 ~ Normal(m*5 + b, sigma)

end

xs = [0,1,2,3]
ys = [0,1,4,6]
n = 4
xs = [i for i in 0:n]
ys = [2*x for x in xs]
println("xs: ", xs)
println("ys: ", ys)
model = linear_regression2(xs,ys)

# chns = sample(model, Prior(), 100_000)
# chns = sample(model, MH(), 100_000)
# chns = sample(model, PG(15), 10_000)
# chns = sample(model, SMC(), 10_000)
# chns = sample(model, IS(), 100_000)
chns = sample(model, NUTS(), 10_000)

display(chns)


"""
    gram_schmidt(A::Array[, ret_coef_mat::Bool])

Return orthogonalized basis.
Optionally returns the Gram-Schmidt coefficient matrix using the ret_coef_mat parameter.

# Example

```julia
julia> x = [
        [ 3  -1   5]
        [-5   2  -1]
        [-3   9   2]
    ]

julia> gram_schmidt(x)
3×3 Array{Float64,2}:
  3.0      -1.0       5.0
 -3.11429   1.37143   2.14286
  2.68728   6.5689   -0.298587

```
"""
function gram_schmidt{T<:AbstractFloat}(A::Array{T, 2}; ret_coef_mat::Bool = false)
    # Assume the vectors are given as row vectors of X
    # Maybe redo later since vectors in Julia are stored in column-major order.

    X = transpose(copy(A))
    X_star = zeros(X)

    # Coefficient matrix
    μ = zeros(X)

    n_cols = size(X)[2]
    for i in 1:n_cols
        X_star[:, i] = X[:, i]
        for j in 1:i-1
            t = dot(X[:, i], X_star[:, j]) / dot(X_star[:, j], X_star[:, j])
            μ[j, i] = t
            X_star[:, i] -= (t * X_star[:, j])
        end
        μ[i,i] = 1
    end

    if ret_coef_mat
        transpose(X_star), transpose(μ)
    else
        transpose(X_star)
    end
end

function gram_schmidt{T<:Integer}(A::Array{T, 2}; ret_coef_mat::Bool = false)
    gram_schmidt(float(A), ret_coef_mat=ret_coef_mat)
end


using Statistics, StatsBase, Random, LinearAlgebra
Random.seed!(1)

function cmHitTime()
    catIndex, mouseIndex, t = 1, 5, 0
        while catIndex != mouseIndex
            catIndex += catIndex == 1 ? 1 : rand([-1,1])
            mouseIndex += mouseIndex == 5 ? -1 : rand([-1,1])
            t += 1
        end
    return t
end
function mcTraj(P,initState,T,stopState=0)
    n = size(P)[1]
    state = initState
    traj = [state]
    for t in 1:T-1
        state = sample(1:n,weights(P[state,:]))
        push!(traj,state)
        if state == stopState
            break
        end
    end
    return traj
end
N = 10^6
P = [   0   1   0   0   0;
        1/4 0   1/4 1/4 1/4;
        0   1/2 0   0   1/2;
        0   1/2 0   0   1/2;
        0   0   0   0   1]

theor = [1 0 0 0] * (inv(I - P[1:4,1:4])*ones(4))
est1 = mean([cmHitTime() for _ in 1:N])
est2 = mean([length(mcTraj(P,1,10^6,5))-1 for _ in 1:N])

P[5,:] = [1 0 0 0 0]
pi5 = sum(mcTraj(P,1,N) .== 5)/N
est3 = 1/pi5 - 1

println("Theoretical: ", theor)
println("Estimate 1: ",est1)
println("Estimate 2: ",est2)
println("Estimate 3: ",est3)


### A Pluto.jl notebook ###
# v0.12.16

using Markdown
using InteractiveUtils

# ╔═╡ e077ab16-22e5-11eb-15ed-5f0654bf7328
using Pkg, DrWatson

# ╔═╡ e75a9a06-22e5-11eb-2e36-8d4f62b830ed
begin
	@quickactivate "StatisticsWithJuliaPlutoNotebooks"
	using Random, Distributions, Plots
	Random.seed!(0)
end;

# ╔═╡ ca5fbdc8-22e5-11eb-0a60-cb0a127a3ca5
md"## Listing5.01"

# ╔═╡ fa68607e-22e5-11eb-0558-c9a4d9f77426
begin
	lambda  = 1/4.5
	expDist = Exponential(1/lambda)
	n, N    = 10, 10^6

	means     = Array{Float64}(undef, N)
	variances = Array{Float64}(undef, N)

	for i in 1:N
		data = rand(expDist,n)
		means[i] = mean(data)
		variances[i] = var(data)
	end
end

# ╔═╡ 78fd898a-2527-11eb-048f-3be84c5d8649
("Actual mean: ",mean(expDist),
		"Mean of sample means: ",mean(means))

# ╔═╡ 78fdc0d2-2527-11eb-3fe8-5738d4af3806
("Actual variance: ",var(expDist),
	"Mean of sample variances: ",mean(variances))

# ╔═╡ 78fe5bf8-2527-11eb-2c19-1d2e7d1277af
begin
	stephist(means, bins=200, c=:blue, normed=true,
		label="Histogram of Sample Means")
	stephist!(variances, bins=600, c=:red, normed=true,
		label="Histogram of Sample Variances", xlims=(0,40), ylims=(0,0.4),
			xlabel = "Statistic value", ylabel = "Density")
end

# ╔═╡ 475ff888-22e6-11eb-2354-09f8f40a8e12
md"## End of listing5.01"

# ╔═╡ Cell order:
# ╟─ca5fbdc8-22e5-11eb-0a60-cb0a127a3ca5
# ╠═e077ab16-22e5-11eb-15ed-5f0654bf7328
# ╠═e75a9a06-22e5-11eb-2e36-8d4f62b830ed
# ╠═fa68607e-22e5-11eb-0558-c9a4d9f77426
# ╠═78fd898a-2527-11eb-048f-3be84c5d8649
# ╠═78fdc0d2-2527-11eb-3fe8-5738d4af3806
# ╠═78fe5bf8-2527-11eb-2c19-1d2e7d1277af
# ╟─475ff888-22e6-11eb-2354-09f8f40a8e12


# coordinates.jl
# Maps having to do with coordinate transforms.

"""
A Cartesion to Polar map. First dimension is interpreted as radial distance, second as an angle.
The unit circle is mapped to a square [-1,1]x[-1,1]

"""
struct CartToPolarMap{T} <: AbstractMap{SVector{2,T}, SVector{2,T}}
end

(m::CartToPolarMap)(x) = applymap(m, x)

applymap(map::CartToPolarMap, x::SVector{2,T}) where {T} = SVector{2,T}(sqrt(x[1]^2+x[2]^2)*2-1, atan(x[2],x[1])/pi)

inv(map::CartToPolarMap{T}) where {T} = PolarToCartMap{T}()

"""
A Polar to Cartesian map. The angle is mapped to the second dimension, radius to the first.
A square [-1,1]x[-1,1] is mapped to the unit circle
"""
struct PolarToCartMap{T} <: AbstractMap{SVector{2,T}, SVector{2,T}}
end

(m::PolarToCartMap)(x) = applymap(m, x)

applymap(map::PolarToCartMap, x::SVector{2,T}) where {T} = SVector{2,T}((x[1]+1)/2*cos(pi*x[2]), (x[1]+1)/2*sin(pi*x[2]))

inv(map::PolarToCartMap{T}) where {T} = CartToPolarMap{T}()


# Cost functions
"""
    mae(ŷ, y)

Return the mean of absolute error `sum(abs.(ŷ .- y)) / length(y)`
"""
mae(ŷ, y) = sum(abs.(ŷ .- y)) * 1 // length(y)

"""
    mse(ŷ, y)

Return the mean squared error `sum((ŷ .- y).^2) / length(y)`.
"""
mse(ŷ, y) = sum((ŷ .- y).^2) * 1 // length(y)

"""
    msle(ŷ, y; ϵ=eps(eltype(ŷ)))

Returns the mean of the squared logarithmic errors `sum((log.(ŷ .+ ϵ) .- log.(y .+ ϵ)).^2) / length(y)`.
The `ϵ` term provides numerical stability.

This error penalizes an under-predicted estimate greater than an over-predicted estimate.
"""
msle(ŷ, y; ϵ=eps(eltype(ŷ))) = sum((log.(ŷ .+ ϵ) .- log.(y .+ ϵ)).^2) * 1 // length(y)

"""
    huber_loss(ŷ, y; δ=1.0)

Computes the mean of the Huber loss given the prediction `ŷ` and true values `y`. By default, δ is set to 1.0.

                    | 0.5*|ŷ - y|,   for |ŷ - y| <= δ
      Hubber loss = |
                    |  δ*(|ŷ - y| - 0.5*δ),  otherwise

[`Huber Loss`](https://en.wikipedia.org/wiki/Huber_loss).
"""
function huber_loss(ŷ, y;  δ=eltype(ŷ)(1))
   abs_error = abs.(ŷ .- y)
   temp = abs_error .<  δ
   x = eltype(ŷ)(0.5)
   hub_loss = sum(((abs_error.^2) .* temp) .* x .+ δ*(abs_error .- x*δ) .* (1 .- temp)) * 1 // length(y)
end

function _crossentropy(ŷ::AbstractVecOrMat, y::AbstractVecOrMat, weight::Nothing)
  return -sum(y .* log.(ŷ)) * 1 // size(y, 2)
end

function _crossentropy(ŷ::AbstractVecOrMat, y::AbstractVecOrMat, weight::Number)
  return -sum(y .* log.(ŷ)) .* weight * 1 // size(y, 2)
end

function _crossentropy(ŷ::AbstractVecOrMat, y::AbstractVecOrMat, weight::AbstractVector)
  return -sum(y .* log.(ŷ) .* weight) * 1 // size(y, 2)
end

"""
    crossentropy(ŷ, y; weight=1)

Return the crossentropy computed as `-sum(y .* log.(ŷ) .* weight) / size(y, 2)`.

See also [`logitcrossentropy`](@ref), [`binarycrossentropy`](@ref).
"""
crossentropy(ŷ::AbstractVecOrMat, y::AbstractVecOrMat; weight=nothing) = _crossentropy(ŷ, y, weight)

"""
    logitcrossentropy(ŷ, y; weight=1)

Return the crossentropy computed after a [softmax](@ref) operation:

  -sum(y .* logsoftmax(ŷ) .* weight) / size(y, 2)

See also [`crossentropy`](@ref), [`binarycrossentropy`](@ref).
"""
function logitcrossentropy(ŷ::AbstractVecOrMat, y::AbstractVecOrMat; weight = 1)
  return -sum(y .* logsoftmax(ŷ) .* weight) * 1 // size(y, 2)
end

"""
    binarycrossentropy(ŷ, y; ϵ=eps(ŷ))

Return `-y*log(ŷ + ϵ) - (1-y)*log(1-ŷ + ϵ)`. The ϵ term provides numerical stability.

Typically, the prediction `ŷ` is given by the output of a [`sigmoid`](@ref) activation.
"""
binarycrossentropy(ŷ, y; ϵ=eps(ŷ)) = -y*log(ŷ + ϵ) - (1 - y)*log(1 - ŷ + ϵ)

# Re-definition to fix interaction with CuArrays.
CuArrays.@cufunc binarycrossentropy(ŷ, y; ϵ=eps(ŷ)) = -y*log(ŷ + ϵ) - (1 - y)*log(1 - ŷ + ϵ)

"""
    logitbinarycrossentropy(ŷ, y)

`logitbinarycrossentropy(ŷ, y)` is mathematically equivalent to `binarycrossentropy(σ(ŷ), y)`
but it is more numerically stable.

See also [`binarycrossentropy`](@ref), [`sigmoid`](@ref), [`logsigmoid`](@ref).
"""
logitbinarycrossentropy(ŷ, y) = (1 - y)*ŷ - logσ(ŷ)

# Re-definition to fix interaction with CuArrays.
CuArrays.@cufunc logitbinarycrossentropy(ŷ, y) = (1 - y)*ŷ - logσ(ŷ)

"""
    normalise(x; dims=1)

Normalises `x` to mean 0 and standard deviation 1, across the dimensions given by `dims`. Defaults to normalising over columns.

```julia-repl
julia> a = reshape(collect(1:9), 3, 3)
3×3 Array{Int64,2}:
  1  4  7
  2  5  8
  3  6  9

julia> normalise(a)
3×3 Array{Float64,2}:
  -1.22474  -1.22474  -1.22474
  0.0       0.0       0.0
  1.22474   1.22474   1.22474

julia> normalise(a, dims=2)
3×3 Array{Float64,2}:
  -1.22474  0.0  1.22474
  -1.22474  0.0  1.22474
  -1.22474  0.0  1.22474
```
"""
function normalise(x::AbstractArray; dims=1)
  μ′ = mean(x, dims = dims)
  σ′ = std(x, dims = dims, mean = μ′, corrected=false)
  return (x .- μ′) ./ σ′
end

"""
    kldivergence(ŷ, y)

KLDivergence is a measure of how much one probability distribution is different from the other.
It is always non-negative and zero only when both the distributions are equal everywhere.

[KL Divergence](https://en.wikipedia.org/wiki/Kullback%E2%80%93Leibler_divergence).
"""
function kldivergence(ŷ, y)
  entropy = sum(y .* log.(y)) * 1 //size(y,2)
  cross_entropy = crossentropy(ŷ, y)
  return entropy + cross_entropy
end

"""
    poisson(ŷ, y)

Poisson loss function is a measure of how the predicted distribution diverges from the expected distribution.
Returns `sum(ŷ .- y .* log.(ŷ)) / size(y, 2)`

[Poisson Loss](https://peltarion.com/knowledge-center/documentation/modeling-view/build-an-ai-model/loss-functions/poisson).
"""
poisson(ŷ, y) = sum(ŷ .- y .* log.(ŷ)) * 1 // size(y,2)

"""
    hinge(ŷ, y)

Measures the loss given the prediction `ŷ` and true labels `y` (containing 1 or -1).
Returns `sum((max.(0, 1 .- ŷ .* y))) / size(y, 2)`

[Hinge Loss](https://en.wikipedia.org/wiki/Hinge_loss)
See also [`squared_hinge`](@ref).
"""
hinge(ŷ, y) = sum(max.(0, 1 .-  ŷ .* y)) * 1 // size(y, 2)

"""
    squared_hinge(ŷ, y)

Computes squared hinge loss given the prediction `ŷ` and true labels `y` (conatining 1 or -1).
Returns `sum((max.(0, 1 .- ŷ .* y)).^2) / size(y, 2)`

See also [`hinge`](@ref).
"""
squared_hinge(ŷ, y) = sum((max.(0, 1 .- ŷ .* y)).^2) * 1 // size(y, 2)

"""
    dice_coeff_loss(ŷ, y; smooth=1)

Loss function used in Image Segmentation. Calculates loss based on dice coefficient. Similar to F1_score.
Returns `1 - 2*sum(|ŷ .* y| + smooth) / (sum(ŷ.^2) + sum(y.^2) + smooth)`

[V-Net: Fully Convolutional Neural Networks forVolumetric Medical Image Segmentation](https://arxiv.org/pdf/1606.04797v1.pdf)
"""
dice_coeff_loss(ŷ, y; smooth=eltype(ŷ)(1.0)) = 1 - (2*sum(y .* ŷ) + smooth) / (sum(y.^2) + sum(ŷ.^2) + smooth)

"""
    tversky_loss(ŷ, y; β=0.7)

Used with imbalanced data to give more weightage to False negatives.
Larger β weigh recall higher than precision (by placing more emphasis on false negatives)
Returns `1 - sum(|y .* ŷ| + 1) / (sum(y .* ŷ + β*(1 .- y) .* ŷ + (1 - β)*y .* (1 .- ŷ)) + 1)`

[Tversky loss function for image segmentation using 3D fully convolutional deep networks](https://arxiv.org/pdf/1706.05721.pdf)
"""
tversky_loss(ŷ, y; β=eltype(ŷ)(0.7)) = 1 - (sum(y .* ŷ) + 1) / (sum(y .* ŷ + β*(1 .- y) .* ŷ + (1 - β)*y .* (1 .- ŷ)) + 1)

"""
    flatten(x::AbstractArray)

Transforms (w,h,c,b)-shaped input into (w x h x c,b)-shaped output,
by linearizing all values for each element in the batch.
"""
function flatten(x::AbstractArray)
  return reshape(x, :, size(x)[end])
end


#=
# 3. Non-uniqueness and discrete circulation

There is no unique solution to the two-dimensional potential flow problem in the presence of an impenetrable surface, since one can choose any value for the uniform value $\mathfrak{s}_0$ and still enforce the no-penetration condition. Equivalently, we can specify any circulation about the body and still enforce this condition. For later use, let us write this uniform surface streamfunction as $\mathfrak{s}_0 = s_0 \mathfrak{1}$, where $s_0$ is a single scalar value and $\mathfrak{1}$ is a vector of ones on all surface points. The discrete circulation $\Gamma_b$ about the body is given by the sum of the bound vortex sheet data and can be written compactly as

$\Gamma_b = \mathfrak{1}^{T} \mathfrak{f}.$

In `GridPotentialFlow.jl`, the user cannot specify $s_0$ directly. Instead, the package solves the system

$\begin{bmatrix}
\mathsf{L} & \mathsf{R} & 0\\
\mathsf{E} &  0 & \mathfrak{1}\\
0 & \mathfrak{1}^{T} & 0\\
\end{bmatrix} \begin{pmatrix} \mathsf{s} \\ \mathfrak{f} \\ s_0 \end{pmatrix} =
\begin{pmatrix} -\mathsf{w} \\ \mathfrak{s}'_b \\ \Gamma_b \end{pmatrix}.$

and the circulation can be specified using the `Γ` keyword for each body. If it is not specified, it is set to zero. The return value of `solvesystem` has a field `ψ₀`, whose entries represent the value for $s_0$ for each body, which are in this case the Lagrange multipliers to enforce the specified circulation.
=#

# We now illustrate how to specify the circulation by simulating the flow around two cylinders that have different circulations.

#md # ```@setup 3.-Non-uniqueness-and-discrete-circulation
#md # using GridPotentialFlow
#md # using Plots
#md # ```
#!md using GridPotentialFlow
#!md using Plots
Δx = 0.01
Lx = 2.0
xlim = (-Lx/2,Lx/2)
ylim = (-Lx/2,Lx/2)
g = PhysicalGrid(xlim,ylim,Δx);

# The two cylinders are positioned using the tools from `RigidTransform` from `RigidBodyTools.jl` and the circulation of the first one is set to a non-zero value.
R = Lx/8;
b_left = PotentialFlowBody(Circle(R,Δx),Γ=1.0)
T = RigidTransform((-Lx/4,0.0),0.0)
T(b_left);
b_right = PotentialFlowBody(Circle(R,Δx))
T = RigidTransform((Lx/4,0.0),0.0)
T(b_right);

# A plot of the streamfunction then clearly shows the non-zero circulation about the left cylinder.
model = VortexModel(g,bodies=[b_left,b_right])
s = streamfunction(model)
plot(s,g)
plot!(b_left,fillcolor=:black,fillrange=0,fillalpha=0.25,linecolor=:black,linewidth=2)
plot!(b_right,fillcolor=:black,fillrange=0,fillalpha=0.25,linecolor=:black,linewidth=2)

#=
When the previously introduced saddle point system

$\begin{bmatrix}
\mathsf{L} & \mathsf{R} \\
\mathsf{E} &  0
\end{bmatrix} \begin{pmatrix} \mathsf{s} \\ \mathfrak{f} \end{pmatrix} =
\begin{pmatrix} -\mathsf{w} \\ \mathfrak{s}'_b - \mathfrak{s}_0 \end{pmatrix}.$

is solved, the resulting circulation is

$\Gamma_{b} = \mathfrak{1}^{T} \mathsf{S}^{-1}\left(\mathfrak{s}_{b}^{\prime}+\mathsf{EL}^{-1} \mathsf{w}\right)-s_{0} \mathfrak{1}^{T} \mathsf{S}^{-1} \mathfrak{1},$

where $\mathsf{S} = -\mathsf{EL}^{-1}\mathsf{R}$ is the Schur complement. The scalar factor $\mathfrak{1}^{T} \mathsf{S}^{-1} \mathfrak{1}$ in this expression is a property of the set of points and their immersion into the Cartesian grid. Part of this factor, $\mathsf{S}^{-1} \mathfrak{1}$, represents the bound vortex sheet strength associated with a uniform, unit-strength streamfunction on the surface. This sheet has a particularly important role in some of the discussion to follow, so we will denote its strength by $\mathfrak{f}_0$:

$\mathfrak{f}_0 \equiv \mathsf{S}^{-1} \mathfrak{1}.$

We demonstrate below the distribution of $\mathfrak{f}_0$ for elliptical cylinders with different aspect ratios.
=#

Δx = 0.004
Lx = 2.0
xlim = (-Lx/2,Lx/2)
ylim = (-Lx/2,Lx/2)
g = PhysicalGrid(xlim,ylim,Δx);

# Let us consider four cylinders with aspect ratios 1, 2, 3, and infinity. An ellipse with infinite aspect ratio is a flat plate, so we construct that case separately.
a = Lx/4 # semi-major axis
AR_list = [1.0,2.0,3.0]
ellipses = [Ellipse(a,a/AR,Δx) for AR in AR_list]
bodies = PotentialFlowBody[]
push!(bodies,[PotentialFlowBody(SplinedBody(hcat(e.x,e.y),3*cellsize(g))) for e in ellipses]...)
push!(bodies,PotentialFlowBody(Plate(2*a,3*Δx)));
colors = [:blue,:red,:green,:black]
plot()
for i in 1:length(bodies)
    plot!(bodies[i],fillrange=0,fillalpha=0.0,linecolor=colors[i],linewidth=2)
end
plot!(xlim=xlim,ylim=ylim,xlabel="x",ylabel="y")

# To compute $\mathfrak{f}_0$, we have to access the internal fields of the associated `system` of our vortexmodel.
f₀_list = []
for i in 1:length(bodies)
    Δs = dlength(bodies[i]);
    model = VortexModel(g,bodies=[bodies[i]]);
    ones = ScalarData(length(bodies[i]))
    ones .= 1.0
    f₀ = model.system.ibp.Sfact\ones
    push!(f₀_list,f₀)
end

# Because the flat plate is an open body, we need to mirror and append its contents to itself to compare it to the ellipses. Furthermore, its bound vortex sheet strength should be divided by two because it represents $\gamma = \gamma^{+} + \gamma^{-}$
append!(f₀_list[end],reverse!(f₀_list[end]));
Δs_plate = dlengthmid(bodies[end]);
append!(Δs_plate,reverse!(Δs_plate));
plot(ylim=(-2,0),xlabel="p/N",title="f₀ for ellipses of different aspect ratios")
for i in 1:length(bodies)-1
    plot!((1:length(f₀_list[i]))/length(f₀_list[i]),f₀_list[i]./dlength(bodies[i]),linecolor=colors[i],label="AR=$(AR_list[i])")
end
plot!((1:length(f₀_list[end]))/length(f₀_list[end]),0.5*f₀_list[end]./Δs_plate,linecolor=colors[end],label="AR=inf")


# # Train Kernel Parameters

# Here we show a few ways to train (optimize) the kernel (hyper)parameters at the example of kernel-based regression using KernelFunctions.jl.
# All options are functionally identical, but differ a little in readability, dependencies, and computational cost.

# We load KernelFunctions and some other packages. Note that while we use `Zygote` for automatic differentiation and `Flux.optimise` for optimization, you should be able to replace them with your favourite autodiff framework or optimizer.

using KernelFunctions
using LinearAlgebra
using Distributions
using Plots
using BenchmarkTools
using Flux
using Flux: Optimise
using Zygote
using Random: seed!
seed!(42);

# ## Data Generation
# We generate a toy dataset in 1 dimension:

xmin, xmax = -3, 3  # Bounds of the data
N = 50 # Number of samples
x_train = rand(Uniform(xmin, xmax), N)  # sample the inputs
σ = 0.1
y_train = sinc.(x_train) + randn(N) * σ  # evaluate a function and add some noise
x_test = range(xmin - 0.1, xmax + 0.1; length=300)
nothing #hide

# Plot the data

scatter(x_train, y_train; label="data")
plot!(x_test, sinc; label="true function")

# ## Manual Approach
# The first option is to rebuild the parametrized kernel from a vector of parameters
# in each evaluation of the cost function. This is similar to the approach taken in
# [Stheno.jl](https://github.com/JuliaGaussianProcesses/Stheno.jl).

# To train the kernel parameters via [Zygote.jl](https://github.com/FluxML/Zygote.jl),
# we need to create a function creating a kernel from an array.
# A simple way to ensure that the kernel parameters are positive
# is to optimize over the logarithm of the parameters.

function kernelcall(θ)
    return (exp(θ[1]) * SqExponentialKernel() + exp(θ[2]) * Matern32Kernel()) ∘
           ScaleTransform(exp(θ[3]))
end
nothing #hide

# From theory we know the prediction for a test set x given
# the kernel parameters and normalization constant:

function f(x, x_train, y_train, θ)
    k = kernelcall(θ[1:3])
    return kernelmatrix(k, x, x_train) *
           ((kernelmatrix(k, x_train) + exp(θ[4]) * I) \ y_train)
end
nothing #hide

# Let's look at our prediction.
# With starting parameters `p0` (picked so we get the right local
# minimum for demonstration) we get:

p0 = [1.1, 0.1, 0.01, 0.001]
θ = log.(p0)
ŷ = f(x_test, x_train, y_train, θ)
scatter(x_train, y_train; label="data")
plot!(x_test, sinc; label="true function")
plot!(x_test, ŷ; label="prediction")

# We define the following loss:

function loss(θ)
    ŷ = f(x_train, x_train, y_train, θ)
    return norm(y_train - ŷ) + exp(θ[4]) * norm(ŷ)
end
nothing #hide

# The loss with our starting point:

loss(θ)

# Computational cost for one step:

@benchmark let
    θ = log.(p0)
    opt = Optimise.ADAGrad(0.5)
    grads = only((Zygote.gradient(loss, θ)))
    Optimise.update!(opt, θ, grads)
end

# ### Training the model

# Setting an initial value and initializing the optimizer:
θ = log.(p0) # Initial vector
opt = Optimise.ADAGrad(0.5)
nothing #hide

# Optimize

anim = Animation()
for i in 1:15
    grads = only((Zygote.gradient(loss, θ)))
    Optimise.update!(opt, θ, grads)
    scatter(
        x_train, y_train; lab="data", title="i = $(i), Loss = $(round(loss(θ), digits = 4))"
    )
    plot!(x_test, sinc; lab="true function")
    plot!(x_test, f(x_test, x_train, y_train, θ); lab="Prediction", lw=3.0)
    frame(anim)
end
gif(anim, "train-kernel-param.gif"; show_msg=false, fps=15);
nothing; #hide

# ![](train-kernel-param.gif)

# Final loss
loss(θ)

# ## Using ParameterHandling.jl
# Alternatively, we can use the [ParameterHandling.jl](https://github.com/invenia/ParameterHandling.jl) package
# to handle the requirement that all kernel parameters should be positive.
# The package also allows arbitrarily nesting named tuples that make the parameters
# more human readable, without having to remember their position in a flat vector.

using ParameterHandling

raw_initial_θ = (
    k1=positive(1.1), k2=positive(0.1), k3=positive(0.01), noise_var=positive(0.001)
)

flat_θ, unflatten = ParameterHandling.value_flatten(raw_initial_θ)
flat_θ #hide

# We define a few relevant functions and note that compared to the previous `kernelcall` function, we do not need explicit `exp`s.

function kernelcall(θ)
    return (θ.k1 * SqExponentialKernel() + θ.k2 * Matern32Kernel()) ∘ ScaleTransform(θ.k3)
end
nothing #hide

function f(x, x_train, y_train, θ)
    k = kernelcall(θ)
    return kernelmatrix(k, x, x_train) *
           ((kernelmatrix(k, x_train) + θ.noise_var * I) \ y_train)
end
nothing #hide

function loss(θ)
    ŷ = f(x_train, x_train, y_train, θ)
    return norm(y_train - ŷ) + θ.noise_var * norm(ŷ)
end
nothing #hide

initial_θ = ParameterHandling.value(raw_initial_θ)
nothing #hide

# The loss at the initial parameter values:

(loss ∘ unflatten)(flat_θ)

# Cost per step

@benchmark let
    θ = flat_θ[:]
    opt = Optimise.ADAGrad(0.5)
    grads = (Zygote.gradient(loss ∘ unflatten, θ))[1]
    Optimise.update!(opt, θ, grads)
end

# ### Training the model

# Optimize

opt = Optimise.ADAGrad(0.5)
for i in 1:15
    grads = (Zygote.gradient(loss ∘ unflatten, flat_θ))[1]
    Optimise.update!(opt, flat_θ, grads)
end
nothing #hide

# Final loss

(loss ∘ unflatten)(flat_θ)

# ## Flux.destructure
# If we don't want to write an explicit function to construct the kernel, we can alternatively use the `Flux.destructure` function.
# Again, we need to ensure that the parameters are positive. Note that the `exp` function is now part of the loss function, instead of part of the kernel construction.

# We could also use ParameterHandling.jl here.
# To do so, one would remove the `exp`s from the loss function below and call `loss ∘ unflatten` as above.

θ = [1.1, 0.1, 0.01, 0.001]

kernel = (θ[1] * SqExponentialKernel() + θ[2] * Matern32Kernel()) ∘ ScaleTransform(θ[3])

params, kernelc = Flux.destructure(kernel);

# This returns the trainable `params` of the kernel and a function to reconstruct the kernel.
kernelc(params)

# From theory we know the prediction for a test set x given
# the kernel parameters and normalization constant

function f(x, x_train, y_train, θ)
    k = kernelc(θ[1:3])
    return kernelmatrix(k, x, x_train) * ((kernelmatrix(k, x_train) + (θ[4]) * I) \ y_train)
end
nothing #hide

function loss(θ)
    ŷ = f(x_train, x_train, y_train, exp.(θ))
    return norm(y_train - ŷ) + exp(θ[4]) * norm(ŷ)
end
nothing #hide

# Cost for one step

@benchmark let θt = θ[:], optt = Optimise.ADAGrad(0.5)
    grads = only((Zygote.gradient(loss, θt)))
    Optimise.update!(optt, θt, grads)
end

# ### Training the model

# The loss at our initial parameter values:
θ = log.([1.1, 0.1, 0.01, 0.001]) # Initial vector
loss(θ)

# Initialize optimizer

opt = Optimise.ADAGrad(0.5)
nothing #hide

# Optimize

for i in 1:15
    grads = only((Zygote.gradient(loss, θ)))
    Optimise.update!(opt, θ, grads)
end
nothing #hide

# Final loss
loss(θ)


# https://turing.ml/dev/tutorials/07-poisson-regression/

# Import Turing, Distributions and DataFrames
using Turing, Distributions, DataFrames, Distributed

# Import MCMCChain, Plots, and StatsPlots for visualizations and diagnostics.
using MCMCChains, Plots, StatsPlots

# Set a seed for reproducibility.
using Random
Random.seed!(99);

# Turn off progress monitor.
Turing.setprogress!(false)

# The true Poisson parameter (mean) of toy data
theta_noalcohol_meds = 1    # no alcohol, took medicine
theta_alcohol_meds = 3      # alcohol, took medicine
theta_noalcohol_nomeds = 6  # no alcohol, no medicine
theta_alcohol_nomeds = 36   # alcohol, no medicine

# no of samples for each of the above cases
q = 100

# Generate data from different Poisson distributions
noalcohol_meds = Poisson(theta_noalcohol_meds)
alcohol_meds = Poisson(theta_alcohol_meds)
noalcohol_nomeds = Poisson(theta_noalcohol_nomeds)
alcohol_nomeds = Poisson(theta_alcohol_nomeds)

nsneeze_data = vcat(rand(noalcohol_meds, q), rand(alcohol_meds, q), rand(noalcohol_nomeds, q), rand(alcohol_nomeds, q))
alcohol_data = vcat(zeros(q), ones(q), zeros(q), ones(q))
meds_data = vcat(zeros(q), zeros(q), ones(q), ones(q))

df = DataFrame(nsneeze = nsneeze_data, alcohol_taken = alchol_data, nomeds_taken = meds_data, product_alcohol_meds = meds_data.*alcohol_data)
df[sample(1:nrow(df), 5, replace = false), :]

# Data Plotting
p1 = Plots.histogram(df[(df[:,:alcohol_taken] .== 0) .& (df[:,:nomeds_taken] .== 0), 1], title = "no_alcohol+meds")
p2 = Plots.histogram((df[(df[:,:alcohol_taken] .== 1) .& (df[:,:nomeds_taken] .== 0), 1]), title = "alcohol+meds")
p3 = Plots.histogram((df[(df[:,:alcohol_taken] .== 0) .& (df[:,:nomeds_taken] .== 1), 1]), title = "no_alcohol+no_meds")
p4 = Plots.histogram((df[(df[:,:alcohol_taken] .== 1) .& (df[:,:nomeds_taken] .== 1), 1]), title = "alcohol+no_meds")
plot(p1, p2, p3, p4, layout = (2, 2), legend = false)

# Convert the DataFrame object to matrices.
data = Matrix(df[:, [:alcohol_taken, :nomeds_taken, :product_alcohol_meds]])
data_labels = df[:, :nsneeze]
data

# Rescale our matrices to help Turing sampler
# in initializing the parameter estimates(normalizing).
data = (data .- mean(data, dims=1)) ./ std(data, dims=1)

@model poisson_regression(x, y, n, σ²) = begin
    b0 ~ Normal(0, σ²)
    b1 ~ Normal(0, σ²)
    b2 ~ Normal(0, σ²)
    b3 ~ Normal(0, σ²)
    for i = 1:n
        theta = b0 + b1*x[i, 1] + b2*x[i,2] + b3*x[i,3]
        y[i] ~ Poisson(exp(theta))
    end
end;

# Retrieve the number of observations.
n, _ = size(data)

# Sample using NUTS.
num_chains = 4
m = poisson_regression(data, data_labels, n, 10)
chain = sample(m, NUTS(200, 0.65), MCMCThreads(), 2_500, num_chains; discard_adapt=false)

# check whether our chains have converged
gelmandiag(chain)

# Taking the first chain
c1 = chain[:,:,1]

# Calculating the exponentiated means
b0_exp = exp(mean(c1[:b0]))
b1_exp = exp(mean(c1[:b1]))
b2_exp = exp(mean(c1[:b2]))
b3_exp = exp(mean(c1[:b3]))

print("The exponent of the meaned values of the weights (or coefficients are): \n")
print("b0: ", b0_exp, " \n", "b1: ", b1_exp, " \n", "b2: ", b2_exp, " \n", "b3: ", b3_exp, " \n")
print("The posterior distributions obtained after sampling can be visualised as :\n")

plot(chain)

# removing warmup samples

# Note the standard deviation before removing the warmup samples
describe(chain)

# Removing the first 200 values of the chains.
chains_new = chain[201:2500,:,:]
describe(chains_new)

plot(chains_new)


# Conjugate

export Conjugate

"""
**Convex conjugate**

    Conjugate(f)

Returns the convex conjugate (also known as Fenchel conjugate, or Fenchel-Legendre transform) of function `f`, that is
```math
f^*(x) = \\sup_y \\{ \\langle y, x \\rangle - f(y) \\}.
```
"""
struct Conjugate{T <: ProximableFunction} <: ProximableFunction
  f::T
  function Conjugate{T}(f::T) where {T<: ProximableFunction}
    if is_convex(f) == false
      error("`f` must be convex")
    end
    new(f)
  end
end

is_prox_accurate(f::Conjugate) = is_prox_accurate(f.f)
is_convex(f::Conjugate) = true
is_cone(f::Conjugate) = is_cone(f.f) && is_convex(f.f)
is_smooth(f::Conjugate) = is_strongly_convex(f.f)
is_strongly_convex(f::Conjugate) = is_smooth(f.f)
is_quadratic(f::Conjugate) = is_strongly_convex(f.f) && is_generalized_quadratic(f.f)
is_generalized_quadratic(f::Conjugate) = is_quadratic(f.f)

fun_dom(f::Conjugate) = fun_dom(f.f)

Conjugate(f::T) where {T <: ProximableFunction} = Conjugate{T}(f)

# only prox! is provided here, call method would require being able to compute
# an element of the subdifferential of the conjugate

function prox!(y::AbstractArray{R}, g::Conjugate, x::AbstractArray{R}, gamma::R=one(R)) where R <: Real
  # Moreau identity
  v = prox!(y, g.f, x/gamma, one(R)/gamma)
  if is_set(g)
    v = zero(R)
  else
    v = dot(x,y) - gamma*dot(y,y) - v
  end
  for k in eachindex(y)
    y[k] = x[k] - gamma*y[k]
  end
  return v
end

# complex case, need to cast inner products to real

function prox!(y::AbstractArray{Complex{T}}, g::Conjugate, x::AbstractArray{Complex{T}}, gamma::R=one(R)) where {R <: Real, T <: RealOrComplex{R}}
  v = prox!(y, g.f, x/gamma, one(R)/gamma)
  if is_set(g)
    v = zero(R)
  else
    v = real(dot(x,y)) - gamma*real(dot(y,y)) - v
  end
  for k in eachindex(y)
    y[k] = x[k] - gamma*y[k]
  end
  return v
end

# naive implementation

function prox_naive(g::Conjugate, x::AbstractArray{T}, gamma::Real=1.0) where T <: RealOrComplex
  y, v = prox_naive(g.f, x/gamma, 1.0/gamma)
  return x - gamma*y, real(dot(x,y)) - gamma*real(dot(y,y)) - v
end


"""
    diffusion_matrix(domain, n)

Assemble discrete diffusion matrix.
"""
function diffusion_matrix end

function diffusion_matrix(domain::ClosedIntervalDomain, n)
    Δx = 1 // n * (domain.right - domain.left)
    inds = [-1, 0, 1]
    stencil = [1, -2, 1] / Δx^2
    diags = [i => fill(s, n + 1 - abs(i)) for (i, s) ∈ zip(inds, stencil)]
    D = spdiagm(diags...)
    D[1, 1] = stencil[1] + stencil[2]
    D[end, end] = stencil[end-1] + stencil[end]
    D
end

function diffusion_matrix(domain::PeriodicIntervalDomain, n)
    Δx = 1 // n * (domain.right - domain.left)
    inds = [-1, 0, 1]
    stencil = [1, -2, 1] / Δx^2
    diags = [i => fill(s, n - abs(i)) for (i, s) ∈ zip(inds, stencil)]
    D = spdiagm(diags...)
    D[1, end] = stencil[1]
    D[end, 1] = stencil[end]
    D
end