### A Pluto.jl notebook ###
# v0.15.1

using Markdown
using InteractiveUtils

# ╔═╡ 61078404-224f-44f1-a75a-9b3537ecc95c
# we write a function that decomposes a square matrix A as A = LU
# where L is lower triangular and U is upper triangular
function LUdecomp(A)

    # we get the size of A
    n, m = size(A)

    # we check if A is indeed a square matrix
    if n != m
        error("Input must be a square matrix")
    end

    # we initialize the output matrices
    L = zeros(n, n)
    U = zeros(n, n)

    # we make a copy of A in order not to modify the original
    B = copy(A)

    for k = 1:n

        # we set the entries of the kth row of U
        for j = k:n
            U[k,j] = B[k,j]
        end

        # we check that we don't divide by zero
        if U[k,k] == 0
            error("** A^(k-1)_{k,k} == 0 in LU decomp")
        end

        # we set the entries of the kth column of L
        for i = k:n
           L[i,k] = B[i,k] / U[k,k]
        end

        # we modify A for the next iteration
        for i = k:n
            for j = k:n
                B[i,j] -= L[i,k] * U[k,j]
            end
        end
    end

    # we return L and U
    return (L, U)
end

# ╔═╡ 4996c2fe-068b-11ec-221d-5fdd71b0f050
md"Simple matrix for which there is no LUdecomp:"

# ╔═╡ 41b2c106-e317-4d65-bb61-94cecb4683a2
A = [0.0 2.0; 3.0 1.0];

# ╔═╡ 75d691e3-34e6-4506-b9b6-559c2412d791
md"This will cause an error:"

# ╔═╡ ffa8307d-751a-42d7-8a5d-ce8a3e92ca0d
LUdecomp(A);

# ╔═╡ a2c516ea-c23f-4f25-8e39-b5213e6fb94e
# we write a function that returns the row index of the (absolutely) largest element in column k of the matrix A
function findLargestInCol(A, k)

	# we get column k
	col = A[:,k]

	# we take elementwise absolute value
	absCol = broadcast(abs, col)

	# we get the index of the largest element
	index = argmax(absCol)

	# we return index
	return index
end

# ╔═╡ ca343c99-8fa8-4489-8c57-3e8afc1fbe71
# we write a more efficient function that swaps rows u and v of some matrix A
function swapRowsV2!(A, u, v)

	# we directly swap rows u and v of A without copying
	temp = A[u, :]
	A[u, :] = A[v, :]
	A[v, :] = temp
end

# ╔═╡ e10c36a3-d0c0-4f63-ac43-f1f04eff1e19
# we write another function that composes A into A = P^{-1}LU where P is stored as a vector and swapRows is more efficient
function PALUdecompV2(A)

    # we get the size of A
    m, n = size(A)

    # we check that A is indeed a square matrix
    if m != n
        error("Input must be a square matrix")
    end

    # we initialize L and U to zero, P to {1,2, ..., n}
    L = zeros(n, n)
    U = zeros(n, n)
    p = collect(1:n)

    # we make a copy of A to work with
    B = copy(A)

    for k = 1:n

        # we find the pivot
        maxk = findLargestInCol(B, k)
        if maxk != k
            swapRowsV2!(B, maxk, k)
            swapRowsV2!(L, maxk, k)

            # we swap two elements of P instead of two rows
            temp = p[maxk]
            p[maxk] = p[k]
            p[k] = temp
        end

        # we proceed with the normal LU decomposition algorithm
        for j = k:n
            U[k,j] = B[k,j]
        end

        # we check that we don't divide by zero
        if U[k,k] == 0
            error("** A^(k-1)_{k,k} == 0 in PALU decomp")
        end

        for i = k:n
           L[i,k] = B[i,k] / U[k,k]
        end

        for i = k:n
            for j = k:n
                B[i,j] -= L[i,k] * U[k,j];
            end
        end
    end

	P = zeros(n,n)
	for i = 1:n
		P[i,p[i]] = 1
	end

    # we return P, L and U
    return P, L, U
end

# ╔═╡ a0f23d95-18a5-427f-b52c-ebdb44df8fa3
PA, LA, UA = PALUdecompV2(A);

# ╔═╡ 23babfbd-d571-454e-8598-3fc9a9044baa
LA*UA == PA*A

# ╔═╡ a5570224-5282-48a3-8f5a-7f5f7a9b741c
md"""
Slightly less trivial example where the error appears in step $k=2$. The problem arises because $B_{22} = B_{21} B_{12} / B_{11}$ so in the usual LU decomposition $A^{1}_{22}$ is zero:
"""

# ╔═╡ 1dd12bff-9da3-4f26-ab3c-23a0e2671aa1
B = [1.0 2.0 3.0; 2.0 4.0 2.0; 1.0 3.0 2.0];

# ╔═╡ 5a34190b-5c77-4732-9134-c6bd432c053c
LUdecomp(B)

# ╔═╡ 751479d0-269c-4be3-893c-1317f9707b5f
PB, LB, UB = PALUdecompV2(B);

# ╔═╡ f5229f99-e78b-48e0-853e-ea1b50aa2633
LB*UB == PB*B

# ╔═╡ 3304fcdb-923f-4b21-932d-cdbfa1c4b243
md"Check that LUdecomp of $PB$ gives the same answer:"

# ╔═╡ b76c8925-323f-423d-98b0-a42bb7ef4f60
LUdecomp(PB*B) == (LB, UB)

# ╔═╡ 344175ec-c622-4967-8f4d-4c1507886f7a
md"Bigger matrix..."

# ╔═╡ 7bfae705-78ff-4f4b-8f85-4f623488d632
C = rand(800, 800);

# ╔═╡ 5c497b8f-d2a3-4609-bd5d-891fe21a4714
PC, LC, UC = PALUdecompV2(C);

# ╔═╡ 5c8e5b61-137e-401b-96f5-df3c9ffaa9dd
md"Estimate 'error' as sum of squares of elements of $LU-PA$:"

# ╔═╡ f76c12c0-3a8e-4ef1-b8dc-a5b2eaf8a761
sum((LC*UC-PC*C).^2)

# ╔═╡ Cell order:
# ╠═61078404-224f-44f1-a75a-9b3537ecc95c
# ╟─4996c2fe-068b-11ec-221d-5fdd71b0f050
# ╠═41b2c106-e317-4d65-bb61-94cecb4683a2
# ╟─75d691e3-34e6-4506-b9b6-559c2412d791
# ╠═ffa8307d-751a-42d7-8a5d-ce8a3e92ca0d
# ╠═a2c516ea-c23f-4f25-8e39-b5213e6fb94e
# ╠═ca343c99-8fa8-4489-8c57-3e8afc1fbe71
# ╠═e10c36a3-d0c0-4f63-ac43-f1f04eff1e19
# ╠═a0f23d95-18a5-427f-b52c-ebdb44df8fa3
# ╠═23babfbd-d571-454e-8598-3fc9a9044baa
# ╟─a5570224-5282-48a3-8f5a-7f5f7a9b741c
# ╠═1dd12bff-9da3-4f26-ab3c-23a0e2671aa1
# ╠═5a34190b-5c77-4732-9134-c6bd432c053c
# ╠═f5229f99-e78b-48e0-853e-ea1b50aa2633
# ╠═751479d0-269c-4be3-893c-1317f9707b5f
# ╟─3304fcdb-923f-4b21-932d-cdbfa1c4b243
# ╠═b76c8925-323f-423d-98b0-a42bb7ef4f60
# ╟─344175ec-c622-4967-8f4d-4c1507886f7a
# ╠═7bfae705-78ff-4f4b-8f85-4f623488d632
# ╠═5c497b8f-d2a3-4609-bd5d-891fe21a4714
# ╟─5c8e5b61-137e-401b-96f5-df3c9ffaa9dd
# ╠═f76c12c0-3a8e-4ef1-b8dc-a5b2eaf8a761


#################### Basis for Polynomial Space ####################3
struct PolynomialBasis{T}
  order::Int
  nodes::Vector{T}      #Quadrature nodes for numerical integration
  weights::Vector{T}    #Quadrature weights for numerical integration
  polynomials::Vector{Poly}
  φ::Matrix{T}     #Vandermonde matrix: basis polynomials evaluated on G-L nodes
  dφ::Matrix{T}
  invφ::Matrix{T}   #inverse of Gen. Vandermonde matrix
end

"""compute Legendre polynomials coefficients, normalized to be orthonormal"""
function poly_legendre{T<:Number}(n, ::Type{T}=Float64, var=:x)
    return poly_jacobi(n,0.0,0.0,T,var)
end

"""compute Jacobi polynomials coefficients, normalized to be orthonormal"""
function poly_jacobi{T<:Number}(n, a, b, ::Type{T}=Float64, var=:x)
    ox = one(T)
    zx = zero(T)
    #Compute initial P_0 and P_1
    γ0 = 2^(a+b+1)/(a+b+1)*gamma(a+1)*gamma(b+1)/gamma(a+b+1);
    p0 = Poly{T}([one(T)/sqrt(γ0)], var)
    if n==0; return p0; end
    γ1 = (a+1)*(b+1)/(a+b+3)*γ0
    p1 = Poly{T}([(a-b)/2/sqrt(γ1), (a+b+2)/2/sqrt(γ1)], var)
    if n==1; return p1; end
    px = Poly{T}([zero(T), one(T)], var)
    aold = 2/(2+a+b)*sqrt((a+1)*(b+1)/(a+b+3))
    for i = 1:(n-1)
        h1 = 2*i+a+b;
        anew = 2/(h1+2)*sqrt((i+1)*(i+1+a+b)*(i+1+a)*(i+1+b)/(h1+1)/(h1+3))
        bnew = -(a^2-b^2)/h1/(h1+2);
        p2 = ox/anew*(-aold*p0 + (px-bnew)*p1);
        aold =anew;
        p0 = p1
        p1 = p2
    end
    return p1
end

function legendre_basis{T<:Number}(order, ::Type{T}=Float64)
  nodes, weights = gausslobatto(order+1)
  φ = zeros(T,order+1,order+1)
  dφ = zeros(T,order+1,order+1)
  polynomials = Vector{Poly}(order+1)
  for n = 0:order
    p = poly_legendre(n, T)
    dp = polyder(p)
    polynomials[n+1] = p
    # Eval interior nodes
    φ[:,n+1] = polyval(p, nodes)
    dφ[:,n+1] = polyval(dp, nodes)
  end
  invφ = inv(φ)
  PolynomialBasis{T}(order,nodes,weights,polynomials,φ,dφ,invφ)
end

"Maps reference coordinates (ξ ∈ [-1,1]) to interval coordinates (x)"
function reference_to_interval(ξ,a::Tuple)
   0.5*(a[2]-a[1])*ξ + 0.5*(a[2]+a[1])
end

"Project function f on polynomial space Vₕ"
function project_function(f, basis, interval::Tuple; component=1)
  nodes = reference_to_interval(basis.nodes, interval)
  f_val = zeros(nodes)
  for i in 1:size(nodes,1)
    f_val[i] = f(nodes[i])[component]
  end
  function model(x,p)
    result = zeros(x)
    for i in 1:size(p,1)
      result.+=p[i]*polyval(basis.polynomials[i], x)
    end
    result
  end
  p0 = zeros(eltype(basis.nodes),basis.order+1)
  curve_fit(model, basis.nodes, f_val, p0)
end

# using FEMBasis
# import FEMBasis: create_basis
# a = legendre_basis(2)
#
# code = create_basis(
# :LegSeg3,
# "3 node linear segment/line orthogonal legendre element",
# NTuple((
#  (-0.7745966692414834,), # N1
#  ( 0.0, ), # N2
#  ( 0.7745966692414834, ), # N3
# )),(
#  :(0.5 + 0.5),
#  :(1.0*u),
#  :(-0.5 + 1.5*u^2),)
# )
# eval(code)
#
# eval_basis!(LegSeg3(),zeros(3,1),(-0.7745966692414834))
# eval_dbasis!(LegSeg3(),zeros(1,3),(-0.7745966692414834))


using GridPotentialFlow
using Plots
Δx = 0.02
Lx = 4.0
xlim = (-Lx/2,Lx/2)
ylim = (-Lx/2,Lx/2)
g = PhysicalGrid(xlim,ylim,Δx);

Rc = Lx/4
Δs = 2Δx
body = PotentialFlowBody(Circle(Rc,Δs))

Rv = 3/2*Rc
Γv = 1.0
v = Vortex(Rv,0.0,Γv);

setΓ(body,-Γv)

model = VortexModel(g,vortices=[v],bodies=[body]);
sol = solve(model);

r1 = sqrt.((body.points.x.-Rv).^2 .+ (body.points.y).^2) # distance from vortex to points on circle
r2 = sqrt.((body.points.x.-Rc^2/Rv).^2 .+ (body.points.y).^2) # distance from image vortex to points on circle
θ1 = π.+atan.((body.points.y)./(body.points.x.-Rv)) # angle from vortex to points on circle
θ2 = atan.((body.points.y),(body.points.x.-Rc^2/Rv)) # angle from image vortex to points on circle
v1x = -v.Γ./(2π*r1).*cos.(θ1.-π/2) # x velocity on circle induced by vortex
v1y = -v.Γ./(2π*r1).*sin.(θ1.-π/2) # y velocity on circle induced by vortex
v2x = v.Γ./(2π*r2).*cos.(θ2.-π/2) # x velocity on circle induced by image vortex
v2y = v.Γ./(2π*r2).*sin.(θ2.-π/2) # y velocity on circle induced by image vortex
V = sqrt.((v1x.+v2x).^2+(v1y.+v2y).^2) # velocity magnitude on circle
γ = -V; # bound vortex sheet strength on circle (velocity on circle is clockwise if positive vortex is to the right of it)

plot(sol.ψ,g)
plot!(body,fillcolor=:black,fillrange=0,fillalpha=0.25,linecolor=:black,linewidth=2)
scatter!([v.x],[v.y],color=:black,markersize=2,xlabel="x",ylabel="y")

plot(sol.f./Δs,label="f/ds",xlabel="body point index")
plot!(γ,label="gamma")

Vv = Γv/(2π*(Rv-Rc^2/Rv))
T = 2π*Rv/Vv

import OrdinaryDiffEq
function rhs(X,model,t)
    setvortexpositions!(model,X)
    Ẋ = vortexvelocities!(model)
    return Ẋ
end
X = getvortexpositions(model)
prob = OrdinaryDiffEq.ODEProblem(rhs,X,(0.0,T),model);
sol = OrdinaryDiffEq.solve(prob,dt=0.1,OrdinaryDiffEq.RK4(),dense=false,adaptive=false);
plot(body,fillcolor=:black,fillrange=0,fillalpha=0.25,linecolor=:black,linewidth=2)
plot!(map(s->s.u[1],sol.u),map(s->s.v[1],sol.u))
scatter!([v.x],[v.y],color=:black,markersize=2,xlabel="x",ylabel="y")

@testset "Vortex near cylinder" begin
    import OrdinaryDiffEq
#jl

    Rv = sqrt(v.x^2+v.y^2)
    Vθ = -v.Γ/(2π*(Rv-Rc^2/Rv))
    Tv = 2π*Rv/abs(Vθ)
#jl

    tspan = (0,Tv)
    model = VortexModel(g,bodies=[body],vortices=[v])
#jl
    function rhs(X,model,t)
        setvortexpositions!(model,X)
        Ẋ = vortexvelocities!(model)
        return Ẋ
    end
#jl
    X = getvortexpositions(model)
    prob = OrdinaryDiffEq.ODEProblem(rhs,X,tspan,model);
    sol = OrdinaryDiffEq.solve(prob,dt=0.1,OrdinaryDiffEq.RK4(),dense=false,adaptive=false);
#jl
    @test isapprox(sol.u[end].u[1], 1.5; atol = 1e-2)
    @test isapprox(sol.u[end].v[1], 0.0; atol = 1e-1)
end

body = PotentialFlowBody(Circle(Rc,Δs), U=(1.0,0.0))
model = VortexModel(g,bodies=[body]);
sol = solve(model);

θ = atan.(body.points.y,body.points.x)
γ = 2*sin.(θ);

plot(sol.ψ,g)
plot!(body,fillcolor=:black,fillrange=0,fillalpha=0.25,linecolor=:black,linewidth=2,xlabel="x",ylabel="y")

plot(sol.f./Δs,label="f/ds",xlabel="body point index")
plot!(γ,label="gamma")

# This file was generated using Literate.jl, https://github.com/fredrikekre/Literate.jl


# # Polynomial Optimization

#md # [![](https://mybinder.org/badge_logo.svg)](@__BINDER_ROOT_URL__/generated/Polynomial Optimization/polynomial_optimization.ipynb)
#md # [![](https://img.shields.io/badge/show-nbviewer-579ACA.svg)](@__NBVIEWER_ROOT_URL__/generated/Polynomial Optimization/polynomial_optimization.ipynb)
# **Contributed by**: Benoît Legat

# ## Introduction

# Consider the polynomial optimization problem [L09, Example 2.2] of
# minimizing the polynomial $x^3 - x^2 + 2xy - y^2 + y^3$
# over the polyhedron defined by the inequalities $x \ge 0, y \ge 0$ and $x + y \geq 1$.

# [L09] Lasserre, J. B.
# *Moments, positive polynomials and their applications*.
# World Scientific, **2009**.

using Test #src
using DynamicPolynomials
@polyvar x y
p = x^3 - x^2 + 2x*y -y^2 + y^3
using SumOfSquares
S = @set x >= 0 && y >= 0 && x + y >= 1
p(x=>1, y=>0), p(x=>1//2, y=>1//2), p(x=>0, y=>1)

# The optimal solutions are $(x, y) = (1, 0)$ and $(x, y) = (0, 1)$ with objective value $0$ but [Ipopt](https://github.com/jump-dev/Ipopt.jl/) only finds the local minimum $(1/2, 1/2)$ with objective value $1/4$.

import Ipopt
model = Model(optimizer_with_attributes(Ipopt.Optimizer, "print_level" => 0))
@variable(model, a >= 0)
@variable(model, b >= 0)
@constraint(model, a + b >= 1)
@NLobjective(model, Min, a^3 - a^2 + 2a*b - b^2 + b^3)
optimize!(model)
@test termination_status(model) == MOI.LOCALLY_SOLVED #src
@show termination_status(model)
@test value(a) ≈ 0.5 rtol=1e-5 #src
@show value(a)
@test value(b) ≈ 0.5 rtol=1e-5 #src
@show value(b)
@test objective_value(model) ≈ 0.25 rtol=1e-5 #src
@show objective_value(model)

# Note that the problem can be written equivalently as follows using [registered functions](https://jump.dev/JuMP.jl/v0.21/nlp/#User-defined-Functions-1).

using Ipopt
model = Model(optimizer_with_attributes(Ipopt.Optimizer, "print_level" => 0))
@variable(model, a >= 0)
@variable(model, b >= 0)
@constraint(model, a + b >= 1)
peval(a, b) = p(x=>a, y=>b)
register(model, :peval, 2, peval, autodiff=true)
@NLobjective(model, Min, peval(a, b))
optimize!(model)
@test termination_status(model) == MOI.LOCALLY_SOLVED #src
@show termination_status(model)
@test value(a) ≈ 0.5 rtol=1e-5 #src
@show value(a)
@test value(b) ≈ 0.5 rtol=1e-5 #src
@show value(b)
@test objective_value(model) ≈ 0.25 rtol=1e-5 #src
@show objective_value(model)

## Sum-of-Squares approach

# We will now see how to find the optimal solution using Sum of Squares Programming.
# We first need to pick an SDP solver, see [here](https://jump.dev/JuMP.jl/v0.21.6/installation/#Supported-solvers) for a list of the available choices.

import CSDP
solver = optimizer_with_attributes(CSDP.Optimizer, MOI.Silent() => true)

# A Sum-of-Squares certificate that $p \ge \alpha$ over the domain `S`, ensures that $\alpha$ is a lower bound to the polynomial optimization problem.
# The following program searches for the largest upper bound and finds zero.

model = SOSModel(solver)
@variable(model, α)
@objective(model, Max, α)
@constraint(model, c3, p >= α, domain = S)
optimize!(model)
@test termination_status(model) == MOI.OPTIMAL #src
@show termination_status(model)
@test objective_value(model) ≈ 0.0 atol=1e-5 #src
@show objective_value(model)

# Using the solution $(1/2, 1/2)$ found by Ipopt of objective value $1/4$
# and this certificate of lower bound $0$ we know that the optimal objective value is in the interval $[0, 1/4]$
# but we still do not know what it is (if we consider that we did not try the solutions $(1, 0)$ and $(0, 1)$ as done in the introduction).
# If the dual of the constraint `c3` was atomic, its atoms would have given optimal solutions of objective value $0$ but that is not the case.

ν3 = moment_matrix(c3)
@test extractatoms(ν3, 1e-3) === nothing #src
extractatoms(ν3, 1e-3) # Returns nothing as the dual is not atomic

# Fortunately, there is a hierarchy of programs with increasingly better bounds that can be solved until we get one with atom dual variables.
# This comes from the way the Sum-of-Squares constraint with domain `S` is formulated.
# The polynomial $p - \alpha$ is guaranteed to be nonnegative over the domain `S` if there exists Sum-of-Squares polynomials $s_0$, $s_1$, $s_2$, $s_3$ such that
# ```math
# p - \alpha = s_0 + s_1 x + s_2 y + s_3 (x + y - 1).
# ```
# Indeed, in the domain `S`, $x$, $y$ and $x + y - 1$ are nonnegative so the right-hand side is a sum of squares hence is nonnegative.
# Once the degrees of $s_1$, $s_2$ and $s_3$ have been decided, the degree needed for $s_0$ will be determined but we have a freedom in choosing the degrees of $s_1$, $s_2$ and $s_3$.
# By default, they are chosen so that the degrees of $s_1 x$, $s_2 y$ and $s_3 (x + y - 1)$ match those of $p - \alpha$ but this can be overwritten using the `maxdegree` keyword argument.

# ### The maxdegree keyword argument

# The maximum total degree (i.e. maximum sum of the exponents of $x$ and $y$) of the monomials of $p$ is 3 so the constraint in the program above is equivalent to `@constraint(model, p >= α, domain = S, maxdegree = 3)`.
# That is, since $x$, $y$ and $x + y - 1$ have total degree 1, the sum of squares polynomials $s_1$, $s_2$ and $s_3$ have been chosen with maximum total degree $2$.
# Since these polynomials are sums of squares, their degree must be even so the next maximum total degree to try is 4.
# For this reason, the keywords `maxdegree = 4` and `maxdegree = 5` have the same effect in this example.
# In general, if the polynomials in the domain are not all odd or all even, each value of `maxdegree` has a different effect in the choice of the maximum total degree of some $s_i$.

model = SOSModel(solver)
@variable(model, α)
@objective(model, Max, α)
@constraint(model, c5, p >= α, domain = S, maxdegree = 5)
optimize!(model)
@test termination_status(model) == MOI.OPTIMAL #src
@show termination_status(model)
@test objective_value(model) ≈ 0.0 atol=1e-5 #src
@show objective_value(model)

# As shown below, for `maxdegree = 5`, the dual variable is atomic as it is the moments of the measure
# $$0.5 \delta(x-1, y) + 0.5 \delta(x, y-1)$$
# where $\delta(x, y)$ is the dirac measure centered at $(0, 0)$.
# Therefore the program provides both a certificate that $0$ is a lower bound and a certificate that it is also an upper bound since it is attained at the global minimizers $(1, 0)$ and $(0, 1)$.

ν5 = moment_matrix(c5)
atoms5 = extractatoms(ν5, 1e-3) #src
@test atoms5.atoms[1].weight ≈ 0.5 rtol=1e-2 #src
@test atoms5.atoms[2].weight ≈ 0.5 rtol=1e-2 #src
@test atoms5.atoms[1].center[2:-1:1] ≈ atoms5.atoms[2].center[1:2] rtol=1e-2 #src
extractatoms(ν5, 1e-3)

# ## A deeper look into atom extraction

# The `extractatoms` function requires a `ranktol` argument that we have set to `1e-3` in the preceding section.
# This argument is used to transform the dual variable into a system of polynomials equations whose solutions give the atoms.
# This transformation uses the SVD decomposition of the moment matrix and discards the equations corresponding to a singular value lower than `ranktol`.
# When this system of equation has an infinite number of solutions, `extractatoms` concludes that the measure is not atomic.
# For instance, with `maxdegree = 3`, we obtain the system
# $$x + y = 1$$
# which contains a whole line of solution.
# This explains `extractatoms` returned `nothing`.

ν3 = moment_matrix(c3)
SumOfSquares.MultivariateMoments.computesupport!(ν3, 1e-3)
@test length(ν3.support.I.p) == 1 #src

# With `maxdegree = 5`, we obtain the system
# ```math
# \begin{aligned}
#   x + y & = 1\\
#   y^2 & = y\\
#   xy & = 0\\
#   x^2 + y & = 1
# \end{aligned}
# ```

ν5 = moment_matrix(c5)
SumOfSquares.MultivariateMoments.computesupport!(ν5, 1e-3)

# This system can be reduced to the equivalent system
# ```math
# \begin{aligned}
#   x + y & = 1\\
#   y^2 & = y
# \end{aligned}
# ```
# which has the solutions $(0, 1)$ and $(1, 0)$.

SemialgebraicSets.computegröbnerbasis!(ideal(ν5.support))
ν5.support
@test length(ν5.support.I.p) == 2 #src

# The function `extractatoms` then reuses the matrix of moments to find the weights $1/2$, $1/2$ corresponding to the diracs centered respectively at $(0, 1)$ and $(1, 0)$.
# This details how the function obtained the result
# $$0.5 \delta(x-1, y) + 0.5 \delta(x, y-1)$$
# given in the previous section.

# ## HomotopyContinuation

# As discussed in the previous section, the atom extraction relies on the solution
# of a system of algebraic equations. The `extractatoms` function takes an optional
# `solver` argument that is used to solve this system of equation.
# If no solver is provided, the default solver of SemialgebraicSets.jl is used which
# currently computes the Gröbner basis, then the multiplication matrices and
# then the Schur decomposition of a random combination of these matrices.
# As the system of equations is obtained from a numerical solution and is represented
# using floating point coefficients, homotopy continuation is recommended as it is
# more numerically robust than Gröbner basis computation.
# The following uses homotopy continuation to solve the system of equations.

using HomotopyContinuation
solver = SemialgebraicSetsHCSolver(; excess_residual_tol = 2e-2, real_tol = 2e-2, compile = false)
atoms5 = extractatoms(ν5, 1e-3, solver) #src
@test length(atoms5.atoms) == 2 #src
@test atoms5.atoms[1].weight + atoms5.atoms[2].weight ≈ 1.0 rtol=1e-2 #src
@test atoms5.atoms[1].center[2:-1:1] ≈ atoms5.atoms[2].center[1:2] rtol=1e-2 #src
extractatoms(ν5, 1e-3, solver)

# As the system has 3 equations for 2 variables and the coefficients of the equations
# are to be treated with tolerance since they originate from the solution of an SDP,
# we need to set `excess_residual_tol` and `real_tol` to a high tolerance otherwise,
# HomotopyContinuation would consider that there is no solution.
# Indeed, as the system is overdetermined (it has more equations than variables)
# HomotopyContinuation expects to have excess solution hence it filters out
# excess solution among the solution found. It determines which solution are in excess
# by comparing the infinity norm of the residuals of the equations at the solution with `excess_residual_tol`.
# It also filters out solution for which the absolute value of the imaginary part of one of the entry
# is larger than `real_tol` and strips out the imaginary part.
# The raw solutions obtained by HomotopyContinuation can be obtained as follows:

F = HomotopyContinuation.System(ν5.support)
res = HomotopyContinuation.solve(F, solver.options...)
r = path_results(res) #src
@test length(r) == 4 #src
@test all(HomotopyContinuation.is_excess_solution, r) #src
path_results(res)

# The printed `residual` above shows why `2e-2` allows to filter how the 2 actual
# solutions from the 2 excess solutions.


n = 10 # rand(1:10)

@test matrixdepot("poisson", Float64, n) == matrixdepot("poisson", n)

A = Matrix(matrixdepot("poisson", n))

n_mesh, n = n, n^2

B = zeros(Float64, n, n)

i = 0
for i_mesh in 1 : n_mesh
    for j_mesh in 1 : n_mesh
        global i = i + 1

        if i_mesh > 1
            j = i - n_mesh
            B[i,j] = -1
        end

        if j_mesh > 1
            j = i - 1
            B[i,j] = -1
        end

        j = i
        B[i,j] = 4

        if j_mesh < n_mesh
            j = i + 1
            B[i,j] = -1
        end

        if i_mesh < n_mesh
            j = i + n_mesh
            B[i,j] = -1
        end
    end
end

@test A ≈ B

println("'poisson' passed test...")


using Distributions

include("confidence_interval.jl")

function one_prop_z_int(p, n, lvl)
    distribution = Normal()

    standard_error = sqrt((p * (1-p)) / n)
    critical_value = quantile(distribution, lvl + (1 - lvl) / 2)

    lowest_value = round(p - critical_value * standard_error, 4)
    highest_value = round(p + critical_value * standard_error, 4)

    return ConfidenceInterval(lowest_value, highest_value)
end

function one_prop_z_test(p, p0, n, alpha, alt)
    distribution = Normal(p0, sqrt((p0 * (1-p0)) / n))

    if alt == "<"
        return round(cdf(distribution, p), 4)
    elseif alt == ">"
        return round(1 - cdf(distribution, p), 4)
    elseif alt == "!="
        if p > p0
            return round(2 * (1 - cdf(distribution, p)), 4)
        else
            return round(2 * cdf(distribution, p), 4)
        end
    else
        return -1
    end
end

function two_prop_z_test(p1, n1, p2, n2, alpha, alt)
    p_pooled = (p1*n1 + p2*n2) / (n1 + n2)
    p_diff = p1 - p2
    distribution = Normal(0, sqrt((p_pooled*(1-p_pooled)/n1) + (p_pooled*(1-p_pooled)/n2)))

    if alt == "<"
        return round(cdf(distribution, p_diff), 4)
    elseif alt == ">"
        return round(1 - cdf(distribution, p_diff), 4)
    elseif alt == "!="
        if p_diff > 0
            return round(2 * (1 - cdf(distribution, p_diff)), 4)
        else
            return round(2 * cdf(distribution, p_diff), 4)
        end
    else
        return -1
    end
end

function two_prop_z_int(p1, n1, p2, n2, lvl)
    standard_error = sqrt(((p1 * (1-p1)) / n1) + ((p2 * (1-p2)) / n2))

    standard_normal = Normal()
    critical_value = quantile(standard_normal, lvl + (1 - lvl) / 2)

    lowest_value = round((p1-p2) - critical_value * standard_error, 4)
    highest_value = round((p1-p2) + critical_value * standard_error, 4)

    return ConfidenceInterval(lowest_value, highest_value)
end

function z_interval(x̄, sx, n, lvl)
    distribution = Normal()

    if n < 30
        println("Conditions weren't met, using t-distribution...\n")
        distribution = TDist(n - 1)
    end

    standard_error = sx / sqrt(n)
    critical_value = quantile(distribution, lvl + (1 - lvl) / 2)

    lowest_value = round(x̄ - critical_value * standard_error, 4)
    highest_value = round(x̄ + critical_value * standard_error, 4)

    return ConfidenceInterval(lowest_value, highest_value)
end

function z_test(x̄, μ0, sx, n, alpha, alt)
    distribution = Normal(μ0, sx / sqrt(n))

    if alt == "<"
        return round(cdf(distribution, x̄), 4)
    elseif alt == ">"
        return round(1 - cdf(distribution, x̄), 4)
    elseif alt == "!="
        if x̄ > μ0
            return round(2 * (1 - cdf(distribution, x̄)), 4)
        else
            return round(2 * cdf(distribution, x̄), 4)
        end
    else
        return -1
    end
end

function two_sample_z_interval(x̄1, sx1, n1, x̄2, sx2, n2, lvl)
    distribution = Normal()

    if n1 < 30 || n2 < 30
        println("Conditions weren't met, using t-distribution...\n")
        distribution = TDist(n1 + n2 - 2)
    end

    standard_error = sqrt((((n1 - 1) * sx1 ^ 2) + ((n2 - 1) * sx2 ^ 2)) / (n1 + n2 - 2)) * sqrt(1 / n1 + 1 / n2)
    critical_value = quantile(distribution, lvl + (1 - lvl) / 2)

    lowest_value = round(x̄1 - x̄2 - critical_value * standard_error, 4)
    highest_value = round(x̄1 - x̄2 + critical_value * standard_error, 4)

    return ConfidenceInterval(lowest_value, highest_value)
end

function two_sample_z_test(x̄1, sx1, n1, x̄2, sx2, n2, lvl, alt)
    z = (x̄1 - x̄2) / sqrt((sx1^2 / n1 + sx2^2 / n2))

    standard_normal = Normal()

    if alt == "<"
        return round(cdf(standard_normal, abs(z)), 4)
    elseif alt == ">"
        return round(1 - cdf(standard_normal, abs(z)), 4)
    elseif alt == "!="
        return round(2 * (1 - cdf(standard_normal, abs(z))), 4)
    else
        return -1
    end
end


function integrate_wrt_normal_pdf(f::Function, μ::Real, σ::Real; npts::Int=15)
    nodes, wts = gausshermite(npts)
    return 1/sqrtπ * sum( f.(sqrt2 .* σ .* nodes .+ μ) .* wts)
end

function gh_chg_of_vars(f, ndswts::NTuple{N}, μ, L) where {N}
    nodes = SVector{N}(first.(ndswts)...)
    wt = prod(last.(ndswts))
    feval = f(sqrt2 .* L*nodes .+ μ)
    return wt*feval
end

function integrate_wrt_normal_pdf(f::Function, μ::AbstractVector, L::AbstractMatrix; npts::Int=15)
    k = length(μ)
    k == checksquare(L) || throw(DomainError())

    ndswts = zip(gausshermite(npts)...)

    # https://stackoverflow.com/questions/56120583/n-dimensional-cartesian-product-of-a-set-in-julia
    quadprod = product( ntuple(i->ndswts, k)...)

    #closure
    g(node_wt_ntup) = gh_chg_of_vars(f, node_wt_ntup, μ, L)

    return (1/sqrtπ)^k * sum( g(vw) for vw in quadprod)
end


# Generate the first n numbers in Halton's sequence with base b.
#
#   seq = haltonsequence(b, n)
#
# Output:
#  seq - sequence of n Halton numbers
#
# Inputs:
#  b - base of the sequence
#  n - length of vector to be generated
#
function haltonsequence(b, n)
  map((i) -> haltonnumber(b, i), 1:n)
end

# Generate the nth Halton number in the sequence with base b.
#
# Notes:
#  Implementation is based on the psudo code in:
#    http://en.wikipedia.org/wiki/Halton_sequence
#
function haltonnumber(base, index)
  res = 0
  f = 1 / base
  i = index

  while (i > 0)
    res += f * (i % base)
    i = ifloor(i / base)
    f = f / base
  end

  return res
end


### A Pluto.jl notebook ###
# v0.12.11

using Markdown
using InteractiveUtils

# ╔═╡ cb23b3e4-30ba-11eb-048b-33940fa83826
using ModelingToolkit

# ╔═╡ 23f048f2-30bb-11eb-2d54-9fb1c6aed0a5
using Latexify

# ╔═╡ 0d2468ee-30be-11eb-0d75-ed21f3712b85
using Plots

# ╔═╡ 44e05d48-30ba-11eb-1820-c92042f7fc1f
md"ModelingToolkit.jl is a modeling language for high-performance symbolic-numeric computation in scientific computing and scientific machine learning. It allows for users to give a high-level description of a model for symbolic preprocessing to analyze and enhance the model. ModelingToolkit can automatically generate fast functions for model components like Jacobians and Hessians, along with automatically sparsifying and parallelizing the computations. Automatic transformations, such as index reduction, can be applied to the model to make it easier for numerical solvers to handle.
"

# ╔═╡ f6b027e8-30ba-11eb-3279-cfb2ba45c50b
@variables x y

# ╔═╡ 0a1f16ba-30bb-11eb-2116-218fecf8c8e5
z = x^3 + y

# ╔═╡ 10315162-30bb-11eb-1462-d3fb963a29ec
latexify(z)

# ╔═╡ 0887b66c-30bc-11eb-0353-93ba2f2fcba1
z1 = t + t^2

# ╔═╡ 2f5af0ba-30bc-11eb-21d7-43f31111d22e
@derivatives D'~t

# ╔═╡ 1ccf0eae-30bc-11eb-1d8a-cd7f727bb69b
D(z1)

# ╔═╡ 40072080-30bc-11eb-2b27-fb501cc055c8
dz1=expand_derivatives(D(z1))

# ╔═╡ 55a84814-30bc-11eb-25df-cff76501368c
latexify(z1)

# ╔═╡ 4d6c6c00-30bc-11eb-1da6-216840f18165
latexify(dz1)

# ╔═╡ c710d79e-30bc-11eb-06df-8f322f48cda0
fz1=build_function(z1,t,expression=false)

# ╔═╡ 881a68a6-30bd-11eb-35c2-4dc061e74ce2
t1=[1:20...]

# ╔═╡ 1b06d722-30bd-11eb-0bbc-27e45fde5a46
ft1=fz1.([1:20...])

# ╔═╡ f8f7d7c2-30bd-11eb-13ee-e9e682866368
plot(t1,ft1)

# ╔═╡ 9e94b908-30bb-11eb-19f0-33840780b3ee
build_function(z,[x,y])

# ╔═╡ a41d0c4a-30bb-11eb-1434-4982edcff617

# ╔═╡ 440889ec-30bb-11eb-3c5c-a519f291111a
A = [x^2+y 0 2x
     0     0 2y
     y^2+x 0 0]

# ╔═╡ 4b7b92b2-30bb-11eb-342f-0fb5b3f27ec5
latexify(A)

# ╔═╡ Cell order:
# ╠═44e05d48-30ba-11eb-1820-c92042f7fc1f
# ╠═cb23b3e4-30ba-11eb-048b-33940fa83826
# ╠═23f048f2-30bb-11eb-2d54-9fb1c6aed0a5
# ╠═0d2468ee-30be-11eb-0d75-ed21f3712b85
# ╠═f6b027e8-30ba-11eb-3279-cfb2ba45c50b
# ╠═0a1f16ba-30bb-11eb-2116-218fecf8c8e5
# ╠═10315162-30bb-11eb-1462-d3fb963a29ec
# ╠═0887b66c-30bc-11eb-0353-93ba2f2fcba1
# ╠═2f5af0ba-30bc-11eb-21d7-43f31111d22e
# ╠═1ccf0eae-30bc-11eb-1d8a-cd7f727bb69b
# ╠═40072080-30bc-11eb-2b27-fb501cc055c8
# ╠═55a84814-30bc-11eb-25df-cff76501368c
# ╠═4d6c6c00-30bc-11eb-1da6-216840f18165
# ╠═c710d79e-30bc-11eb-06df-8f322f48cda0
# ╠═881a68a6-30bd-11eb-35c2-4dc061e74ce2
# ╠═1b06d722-30bd-11eb-0bbc-27e45fde5a46
# ╠═f8f7d7c2-30bd-11eb-13ee-e9e682866368
# ╠═9e94b908-30bb-11eb-19f0-33840780b3ee
# ╠═a41d0c4a-30bb-11eb-1434-4982edcff617
# ╠═440889ec-30bb-11eb-3c5c-a519f291111a
# ╠═4b7b92b2-30bb-11eb-342f-0fb5b3f27ec5


# ---
# title: 1523. Count Odd Numbers in an Interval Range
# id: problem1523
# author: zhwang
# date: 2022-03-18
# difficulty: Easy
# categories: Math
# link: <https://leetcode.com/problems/count-odd-numbers-in-an-interval-range/description/>
# hidden: true
# ---
#
# Given two non-negative integers `low` and `high`. Return the _count of odd
# numbers between_`low` _and_` high` _  (inclusive)_.
#
#
#
# **Example 1:**
#
#
#
#     Input: low = 3, high = 7
#     Output: 3
#     Explanation: The odd numbers between 3 and 7 are [3,5,7].
#
# **Example 2:**
#
#
#
#     Input: low = 8, high = 10
#     Output: 1
#     Explanation: The odd numbers between 8 and 10 are [9].
#
#
#
# **Constraints:**
#
#   * `0 <= low <= high <= 10^9`
#
#
## @lc code=start
using LeetCode

count_odd_numbers(low::Int, high::Int) = (high - low + 1 + (low & 1)) >> 1

## @lc code=end

## @lc test=start
@testset "1523.count-odd-numbers-in-an-interval-range.jl" begin
    @test count_odd_numbers(3, 7) == count_odd_numbers(3, 8) == 3
    @test count_odd_numbers(2, 7) == 3
    @test count_odd_numbers(0, 10 ^ 9) == 5 * 10 ^ 8
end
## @lc test=end

## @lc info=start
# link: [solution to 1523]()
## @lc info=end


export Tanh

"""
`Tanh([domainType=Float64::Type,] dim_in::Tuple)`

Creates an hyperbolic tangent non-linear operator with input dimensions `dim_in`:
```math
\\text{tanh} ( \\mathbf{x} ).
```

"""
struct Tanh{T,N} <: NonLinearOperator
	dim::NTuple{N,Int}
end

function Tanh(DomainType::Type, DomainDim::NTuple{N,Int}) where {N}
	Tanh{DomainType,N}(DomainDim)
end

Tanh(DomainDim::NTuple{N,Int}) where {N} = Tanh{Float64,N}(DomainDim)
Tanh(DomainDim::Vararg{Int}) = Tanh{Float64,length(DomainDim)}(DomainDim)

function mul!(y::AbstractArray{T,N}, L::Tanh{T,N}, x::AbstractArray{T,N}) where {T,N}
	y .= tanh.(x)
end

function mul!(y::AbstractArray,
              J::AdjointOperator{Jacobian{A,TT}},
              b::AbstractArray) where {T,N, A<: Tanh{T,N}, TT <: AbstractArray{T,N}}
    L = J.A
    y .= conj.(sech.(L.x).^2).*b
end

fun_name(L::Tanh) = "tanh"

size(L::Tanh) = (L.dim, L.dim)

domainType(L::Tanh{T,N}) where {T,N} = T
codomainType(L::Tanh{T,N}) where {T,N} = T


"""
    convex_quadratic(Q, b)

Returns a convex quadratic function `fdf` with signature for `DescentMethods.strong_backtracking!`

    fdf(x, α, d) = f(x + α * d), ∇f(x + α * d)

where

      f(x) = 1/2 xᵀQx - bᵀx
     ∇f(x) = Qx - b
    ∇²f(x) = Q (hessian)

For f(x) to be convex, `Q` must be positive semi-definite matrix.
"""
function convex_quadratic(Q::AbstractMatrix, b::AbstractVector)
    @assert isposdef(Q)

    function f(x::AbstractVector, α::Number, d::AbstractVector)
        y = x + α * d
        y = convert(Vector{Float64}, y)
        val = 0.5 * y' * Q * y - b' * y
        grad = Q * y - b
        return (val, grad)
    end
    return f
end

function random_convex_quadratic(dim::Int)
    Q = rand(test_rng, Float64, (dim, dim))
    Q = Q * Q'  # QQᵀ is always positive semi-definite
    b = rand(test_rng, Float64, (dim, ))
    fdf = convex_quadratic(Q, b)
    return fdf, Q, b
end

"""
    convex_test(fdf, x0, Q[, d])

Perform linear search on a convex quadratic function `fdf` with hessian `Q`.
The search starts at `x0` with descent direction `d`.
Default `d` is opposite to gradient of `fdf`.

Return a tuple `(α, α₀)` where `α` is line search minimizer and `α₀` is analytical minimizer.

# Todo
1. Add randomness to default `d` vector.
"""
function convex_test(fdf, x0, Q, d=nothing)
    y0, grad0 = fdf(x0, 0, x0)  # f(x0 + α x0) = f(x0)
    if isnothing(d)
        d = - 0.01 * grad0  # for greater values of d ⟹ α ≉ α₀
    end

    α = Downhill.strong_backtracking!(fdf, x0, d, y0, grad0)

    "Exact minimizer of convex quadratic function. See Nocedal p. 56, eq. 3.55."
    α₀ = - (grad0' * d) / (d' * Q * d)

    return (α, α₀)
end

@testset "Convex quadratic" begin
    dim = 10
    fdf, Q, _ = random_convex_quadratic(dim)
    for i in 1:20
        x0 = rand(test_rng, Float64, (dim, ))
        α, α₀ = convex_test(fdf, x0, Q)
        println(α, '\t', α₀)
        @test isapprox(α, α₀, rtol=0.05)
    end
end


function fixedpointmap(f, x_0, tolerance, maxiter)
    D(f) = x -> ForwardDiff.derivative(f, x)
    f_prime = D(f)
    x_old = x_0
    normdiff = Inf
    iter = 1
    while normdiff > tolerance && iter <= maxiter
        x_new = x_old - f(x_old)/f_prime(x_old)
        normdiff = norm(x_new - x_old)
        x_old = x_new
        iter = iter + 1
    end
    return (value = x_old, normdiff = normdiff, iter = iter)
end

# define a map and parameters
f(x) = (x-1)^2

sol = fixedpointmap(f, 0, 1.0E-8, 1000)
println("Fixed point = $(sol.value), and |x_t+1 - x_t| = $(sol.normdiff) in $(sol.iter) iterations")


"""
2520 is the smallest number that can be divided by each of the numbers from 1 to 10 without any remainder.

What is the smallest positive number that is evenly divisible by all of the numbers from 1 to 20?
"""

ismultiple(x::Int,r::UnitRange) = all([x%i == 0 for i in r])

function smallestmutiple(nmin,nmax)
    r = collect(nmin:nmax)
    multiples = []

    for i in eachindex(r)   # store only the common divisor
        t = r[i]
        t == 1 && continue
        push!(multiples,r[i])

        for j in i:length(r)
            if r[j] % t == 0
                r[j] = r[j] / t
            end
        end
    end
    return prod(multiples)
end

@assert smallestmutiple(1,10) == 2520
@show smallestmutiple(1,20)


# ------------------------------------------------------------------
# Licensed under the MIT License. See LICENSE in the project root.
# ------------------------------------------------------------------

"""
    signarea(A, B, C)

Compute signed area of triangle formed
by points `A`, `B` and `C`.
"""
function signarea(A::Point{2}, B::Point{2}, C::Point{2})
  ((B - A) × (C - A)) / 2
end

"""
    sideof(point, segment)

Determines on which side of the oriented `segment`
the `point` lies. Possible results are `:LEFT`,
`:RIGHT` or `:ON` the segment.
"""
function sideof(p::Point{2,T}, s::Segment{2,T}) where {T}
  a, b = vertices(s)
  area = signarea(p, a, b)
  ifelse(area > atol(T), :LEFT, ifelse(area < -atol(T), :RIGHT, :ON))
end

"""
    sideof(point, chain)

Determines on which side of the closed `chain` the
`point` lies. Possible results are `:INSIDE` or
`:OUTSIDE` the chain.
"""
function sideof(p::Point{2,T}, c::Chain{2,T}) where {T}
  w = windingnumber(p, c)
  ifelse(isapprox(w, zero(T), atol=atol(T)), :OUTSIDE, :INSIDE)
end


"""

Contains embedding of data under diffusion maps.
"""
struct DiffMap
    Y  # m x n
    L
    Λ
    Φ
end

"""

"""
function DiffMap(X; m::Integer=2, σ = 2)
    D, n = size(X)

    # # Step 1: Compute similarity matrix
    𝒦 = gaussian_similarity(X, σ)

    # Step 2: compute laplacian
    L = rlap(𝒦)

    # Step 3: Compute eigenvectors of L
    # Arpack returns complex-valued eigenvalues.
    Λ, Φ = eigs(L, nev=m+1, which=:SR)
    Λ = real(Λ)
    Φ = real(Φ)

    # Remember that (1,...,1) is an eigenvector with eigenvalue 0. This is uninformative,
    # hence we discard it as usual.
    Φ ./= Φ[:, 1]
    Φ = Φ[:, 2:m+1]
    Λ = Λ[2:m+1]
    Y = permutedims(Φ .* reshape(Λ, (1, :)))
    return DiffMap(Y, L, Λ, Φ)
end

getm(DM::DiffMap) = size(DM.Y, 1)
getn(DM::DiffMap) = size(DM.Y, 2)
size(DM::DiffMap) = size(DM.Y)
size(DM::DiffMap, d::Integer) = size(DM.Y, d)

"""
    Return measure of density for laplacian. (#nonzeros / #total)
"""
sparsity(DM::DiffMap) = sparsity(DM.L)
sparsity(A) = sum(iszero.(A)) / length(A)

function Base.iterate(DM::DiffMap, state = 1)
    if state > 4
        return nothing
    else
        getfield(DM, state), state + 1
    end
end


# # Logistic regression

import MaximumLikelihoodProblems

import Distributions
import ForwardDiff
import LogDensityProblems
import StatsFuns
import TransformVariables

struct LogisticRegression{Ty, TX}
    y::Ty
    X::TX
end

function (problem::LogisticRegression)(θ)
    y = problem.y
    X = problem.X

    β = θ.β

    η = X * β
    μ = StatsFuns.logistic.(η)
    log_likelihood = sum(Distributions.logpdf.(Distributions.Bernoulli.(μ), y))
    return log_likelihood
end

N = 10_000

## the first column (the column of all ones) is the intercept
X = hcat(ones(N), randn(N))

size_β = (2,)
β_true = [1.0, 2.0]
η_true = X * β_true
μ_true = StatsFuns.logistic.(η_true)
y = rand.(Distributions.Bernoulli.(μ_true))

problem = LogisticRegression(y, X)
transformation = TransformVariables.as((β = TransformVariables.as(Array, size_β), ))
transformed_problem = LogDensityProblems.TransformedLogDensity(transformation,
                                                               problem)
transformed_gradient_problem = LogDensityProblems.ADgradient(:ForwardDiff,
                                                             transformed_problem)

β_hat_initial_guess = zeros(size_β)
θ_hat_initial = (; β = β_hat_initial_guess)

# θ_hat:

θ_hat = MaximumLikelihoodProblems.fit(transformed_gradient_problem,
                                      θ_hat_initial)

# β_hat:

β_hat = θ_hat[:β]

# Value of the log likelihood function evaluated at θ_hat:

MaximumLikelihoodProblems.loglikelihood(transformed_gradient_problem, θ_hat)


# Code from https://github.com/toddleo/Outliers.jl/blob/master/outliers.jl, with modification

function outliers_iqr(x::Vector{T} where T <: Real; bar::Float64 = 1.5)
    Q1, Q3 = quantile(x, 0.25), quantile(x, 0.75)
    LIF = Q1 - range * (Q3 - Q1)
    UIF = Q3 + range * (Q3 - Q1)

    .!(LIF .< x .< UIF)
end

function outliers_sd(x::Vector{T} where T <: Real; bar::Float64 = 3.0)
    x = zscore(x)
    μ,σ = mean(x), std(x)

    .!((-1 * abs(bar)) .< (x .- μ) ./  σ .< abs(bar))
end

function outliers(
    x::Vector{T} where T <: Real;
    bar::Union{Float64, Nothing} = nothing,
    method::Symbol = :sd
)
    if method == :sd
        outliers_sd(x, bar = isnothing(bar) ? 3.0 : bar)
    elseif method == :iqr
        outliers_iqr(x, bar = isnothing(bar) ? 1.5 : bar)
    else
        throw(ArgumentError("invalid method"))
    end
end


# My formulation

#=
@sym begin
  LagrangePolynomial(xi_, yi_) := Module([N, sum, num, den],
    begin
      N = Length(xi)
      sum = 0
      For( i=1, i <= N, Increment(i),
        begin
          num = 1.0
          den = 1.0
          For( j=1, j <= N, Increment(j),
            begin
              num = If(j != i, num = num * (x-xi[j]), num)
              den = If(j != i, den = den * (xi[i]-xi[j]), den)
            end
          )
          sum = sum + yi[i] * num/den
        end
      ),
      Return(Simplify(sum))
    end
  )
end
=#

# John's better formulation, much better, need to update tests

@sym begin
  LagrangePolynomial(xi_, yi_) := Module([N, sum, num, den],
    begin
      N = Length(xi)
      sum = 0
      For( i=1, i <= N, Increment(i),
        begin
          num = 1
          den = 1
          For( j=1, j <= N, Increment(j),
            begin
              If(j != i, num *= (x-xi[j]))
              If(j != i, den *= (xi[i]-xi[j]))
            end
          )
          sum += yi[i] * num/den
        end
      ),
      Return(Simplify(sum))
    end
  )
end


using PyPlot, PyCall

function euler(f, y0, h, N, t0=0.0)
    t = t0 .+ h*(0:N)
    y = zeros(length(y0), N+1)

    y[:,1] .= y0
    for n = 1:N
        y[:,n+1] = y[:,n] + h * f(t[n], y[:,n])
    end

    return t,y
end

f(t,y) = -y .+ sin(t)
t,y = euler(f, 1, 0.2, 20)
plot(t, y[:], "-o");

yexact(t) = exp(-t) + (sin(t) - cos(t) + exp(-t)) / 2
tt = 0:0.01:2
plot(tt, yexact.(tt))
for h = [0.5, 0.2, 0.1]
    t,y = euler(f, 1, h, round(Int, 2/h))
    plot(t, y[:], "-o")
end
legend(("Exact", "h=0.5", "h=0.2", "h=0.1"));

function rk4(f, y0, h, N, t0=0)
    t = t0 .+ h*(0:N)
    y = zeros(length(y0), N+1)

    y[:,1] .= y0
    for n = 1:N
        k1 = h * f(t[n], y[:,n])
        k2 = h * f(t[n] + h/2, y[:,n] + k1/2)
        k3 = h * f(t[n] + h/2, y[:,n] + k2/2)
        k4 = h * f(t[n] + h, y[:,n] + k3)
        y[:,n+1] = y[:,n] + (k1 + 2k2 + 2k3 + k4) / 6
    end

    return t,y
end

yexact(t) = exp(-t) + (sin(t) - cos(t) + exp(-t)) / 2
tt = 0:0.01:2
plot(tt, yexact.(tt))
for h = [0.5, 0.2, 0.1]
    t,y = rk4(f, 1, h, round(Int, 2/h))
    plot(t, y[:], "-o")
end
legend(("Exact", "h=0.5", "h=0.2", "h=0.1"));

errors = @. y[:] - yexact(t[:])
max_error = maximum(abs.(errors))


module Quadrature
using RandomNumbers

# Functions we will integrate
f1(x) = (1.0 / sqrt(2 * pi)) * exp((-x^2.0) / 2.0)
f1lim = (0.0, 1.0)

f2(x) = ((sin(2.0 * x))^2.0)/(x^2.0)
f2lim = (pi, 2 * pi)

f3(x) = x^(-.5)
f3lim = (0.0, 2.0)

# Change of variable for f1 -> Inf
f4(x) = f1(-log(x)) / x
f4lim = (0.0, e^-f1lim[1])

# Infinite limits of integration for printing
f1inflim = (0.0, Inf)

# PDF of the normal distribution
function normpdf(x, mu=0, sigma=1)
	return (sigma * sqrt(2 * pi))^-1.0 * e^(-(x - mu)^2 / (2 * sigma^2))
end

# Limits to integrate normpdf over
normlim = (-50.0, 50.0)

# PDF of the Cauchy distribution
cauchypdf(x) = (pi * (1.0 + x^2.0))^-1.0

# PDF of the t-distribution
function tpdf(x, dof)
	term1 = gamma(.5 * (dof + 1.0)) / (sqrt(dof * pi) * gamma(dof / 2.0))
	term2 = (1.0 + (x^2.0)/dof)^(-.5*(dof+1.0))
	return term1 * term2
end

# Generate a single random t variable
function rn_t(dof)
	z = rn_standard_normal()
	v = rn_chi_sq(dof)
	return z / sqrt(v / dof)
end

# Utility function to print each entry in an array on its own line
mapln(x) = map(println, x)

# Utility function to truncate
function truncate(x::Float64, minvalue::Float64, maxvalue::Float64)
	if x > maxvalue
		return maxvalue
	elseif x < minvalue
		return minvalue
	else
		return x
	end
end

# Truncated Cauchy
rn_trunc_cauchy(a, b) = truncate(rn_standard_cauchy(), a, b)
rn_trunc_cauchy(a, b, n) = [rn_trunc_cauchy(a, b) for x in 1:n]

# Truncated t
rn_trunc_t(dof, a, b) = truncate(rn_t(dof), a, b)
rn_trunc_t(dof, a, b, n) = [rn_trunc_t(dof, a, b) for x in 1:n]

# Extended trapezoidal rule
function exttrap(f, a, b; nmax=20, epsilon=1e-6)
	# Initialize I
	n = 2
	I = [(b - a) * .5 * ( f(a) + f(b))]

	while n < nmax
		# Determine new points to evaluate at
		irange = 1:2^(n-2)
		evalpoints = map(x -> (a + (2*x - 1) * (b - a) / (2^(n -1))), irange)

		# Sum function evals at new points
		sumterm = sum(map(f, evalpoints))

		# Add new I onto results vector
		push!(I, .5 * I[n-1] + (sumterm * (b - a) / (2^(n-1))))

		# Check for convergence
		if abs(I[n] - I[n-1]) <= epsilon * abs(I[n-1])
			return I
		end

		n += 1
	end

	return I
end

# Romberg integration
function romberg(f, a, b; nmax=20, epsilon=1e-6)
	# Get the results of the trapezoidal rule
	traps = exttrap(f, a, b, nmax=nmax, epsilon=epsilon)

	# Find a size for our results
	resultsize = min(nmax, length(traps))

	# Create a zero matrix and assign traps to the first column
	r = zeros(Float64, (resultsize, resultsize))
	r[:, 1] = traps[1:resultsize]

	# For each row past the first
	for i in 2:resultsize
		# For each column past the first
		for j in 2:i
			r[i,j] = (1 / (4^(j - 1) - 1)) * (4^(j-1) * r[i,j-1] - r[i-1,j-1])
		end

		# Check stopping criteria
		if abs(r[i,i] - r[i,i-1]) <= epsilon * abs(r[i,i-1])
			return r[1:i,1:i]
		end
	end
end

# Extended midpoint rule
function midpoint(f, a, b; nmax=20, epsilon=1e-6)
	# Initialize I
	n = 2
	I = [f(.5 * (a + b))]

	while n < nmax
		# Determine new points to evaluate at:
		# Keep odd numbers from 1:2*3^(n-1) that aren't divisible by 3
		newodds = filter(x -> x % 3 != 0, [1:2:2*3^(n-1)])

		# Divide by (2*3^(n-1)) and scale to (a,b)
		newpoints = a + (b - a) * newodds / (2*3^(n - 1))

		# Sum function evals at this point
		sumterm = sum(map(f, newpoints))

		# Add new I onto results vector
		push!(I, I[n-1] / 3.0 + sumterm * (b - a) / 3.0^(n-1))

		# Check for convergence
		if abs(I[n] - I[n-1]) <= epsilon * abs(I[n-1])
			return I
		end

		n += 1
	end

	return I
end

# Monte Carlo Integration with antithetic variables
function mci(f, a, b; nmax=1e4, epsilon=1e-6)

	# Store our random variables in a vector
	x = [rn_unif(a, b)]

	# Calculate our initial result
	I = [(b - a) * .5 * (f(x[1]) + f(a + b - x[1]))]

	# Initialize n
	n = 2
	while n < nmax
		# Generate a new random variable and add it to the vector
		push!(x, rn_unif(a, b))

		# Calculate new result and push it onto the results vector
		push!(I, (b - a) * .5 * mean(map(f, x) + map(f, a + b - x)))

		# Check for convergence
		if abs(I[n] - I[n-1]) <= epsilon * abs(I[n-1])
			return I
		end

		n += 1
	end

	return I
end

# Monte Carlo Integration with Importance Sampling (Uniform)
function mciunif(f, a, b; n=1000)
	# Store our random variables in a vector
	x = rn_unif(a, b, n)

	# Calculate our estimate
	I = (b - a) * mean(map(f, x))

	return I
end

# Monte Carlo Integration with Importance Sampling (Truncated Standard Cauchy)
function mcicauchy(f, a, b; n=1000)
	# Store our random variables in a vector
	x = rn_trunc_cauchy(a, b, n)

	# Calculate our estimate
	I = mean(map(f, x) ./ map(cauchypdf, x))

	return I
end

# Monte Carlo Integration with Importance Sampling (Truncated t-distribution)
function mcit(f, a, b; n=1000)
	# Use 30 degrees of freedom - in the future this might be worth parameterizing
	dof = 30

	# Store our random variables in a vector
	x = rn_trunc_t(dof, a, b, n)

	# Calculate our estimate
	I = mean(map(f, x) ./ map(x -> tpdf(x, dof), x))

	return I
end

# Run integration function f n times and write final results to a csv file
function mcincsv(f, n, filename)
	println("Writing $n runs to $filename")
	results = [f(x) for x in 1:n]
	writecsv(filename, results)
end

end


# # Lyapunov Function Search

#md # [![](https://mybinder.org/badge_logo.svg)](@__BINDER_ROOT_URL__/generated/Lyapunov Function Search.ipynb)
#md # [![](https://img.shields.io/badge/show-nbviewer-579ACA.svg)](@__NBVIEWER_ROOT_URL__/generated/Lyapunov Function Search.ipynb)
# **Adapted from**: SOSTOOLS' SOSDEMO2 (See Section 4.2 of [SOSTOOLS User's Manual](http://sysos.eng.ox.ac.uk/sostools/sostools.pdf))

using Test #src
using DynamicPolynomials
@polyvar x[1:3]

# We define below the vector field ``\text{d}x/\text{d}t = f``

f = [-x[1]^3 - x[1] * x[3]^2,
     -x[2] - x[1]^2 * x[2],
     -x[3] - 3x[3] / (x[3]^2 + 1) + 3x[1]^2 * x[3]]

# We need to pick an SDP solver, see [here](https://jump.dev/JuMP.jl/v0.21.6/installation/#Supported-solvers) for a list of the available choices.
# We use `SOSModel` instead of `Model` to be able to use the `>=` syntax for Sum-of-Squares constraints.

using SumOfSquares
using CSDP
solver = optimizer_with_attributes(CSDP.Optimizer, MOI.Silent() => true)
model = SOSModel(solver);

# We are searching for a Lyapunov function $V(x)$ with monomials $x_1^2$, $x_2^2$ and $x_3^2$.
# We first define the monomials to be used for the Lyapunov function:

monos = x.^2

# We now define the Lyapunov function as a polynomial decision variable with these monomials:

@variable(model, V, Poly(monos))

# We need to make sure that the Lyapunov function is strictly positive.
# We can do this with a constraint $V(x) \ge \epsilon (x_1^2 + x_2^2 + x_3^2)$,
# let's pick $\epsilon = 1$:

@constraint(model, V >= sum(x.^2))

# We now compute $\text{d}V/\text{d}x \cdot f$.

using LinearAlgebra # Needed for `dot`
dVdt = dot(differentiate(V, x), f)

# The denominator is $x[3]^2 + 1$ is strictly positive so the sign of `dVdt` is the
# same as the sign of its numerator.

P = dVdt.num

# Hence, we constrain this numerator to be nonnegative:

@constraint(model, P <= 0)

# The model is ready to be optimized by the solver:

JuMP.optimize!(model)

# We verify that the solver has found a feasible solution:

JuMP.primal_status(model)
@test JuMP.primal_status(model) == MOI.FEASIBLE_POINT #src

# We can now obtain this feasible solution with:

value(V)
@test iszero(removemonomials(value(V), monos)) #src


using Pkg
Pkg.activate(@__DIR__)
Pkg.instantiate()
##------------------------  ------------------------##
using Ipopt
using LinearAlgebra, ForwardDiff, Zygote
##------------------------  ------------------------##
# Implement hs071 from IPOPT/IPOPT.jl documentation using
# analytical and automatic differentiation.

# min x1 * x4 * (x1 + x2 + x3) + x3
# st  x1 * x2 * x3 * x4 >= 25
#     x1^2 + x2^2 + x3^2 + x4^2 = 40
#     1 <= x1, x2, x3, x4 <= 5
# Start at (1,5,5,1)
# End at (1.000..., 4.743..., 3.821..., 1.379...)

# solution/starting point = x
# objective function = f(x)
# constraints = g(x)

# number of variables
n = 4

# bounds
# 1 <= x1, x2, x3, x4 <= 5
# lower bounds
x_L = [1.0, 1.0, 1.0, 1.0]
# upper bounds
x_U = [5.0, 5.0, 5.0, 5.0]

# number of constraints, dim of g(x)
m = 2

# constraints lower bounds
# x1 * x2 * x3 * x4 >= 25
# x1^2 + x2^2 + x3^2 + x4^2 = 40
g_L = [25.0, 40.0]
# constraints upper bounds
g_U = [2.0e19, 40.0]# unbounded first term

# initial point
x_0 = [1.0, 5.0, 5.0, 1.0]

##---------------------- obj/constrain functions ----------------------##
# objective function
function eval_f(x)
    return x[1] * x[4] * (x[1] + x[2] + x[3]) + x[3]
end

# constraint function
function eval_g(x, g)
    g[1] = x[1] * x[2] * x[3] * x[4]
    g[2] = x[1]^2 + x[2]^2 + x[3]^2 + x[4]^2
    return g
end

function eval_g_ad(x, g)
    g = eltype(x).(g)
    g[1] = x[1] * x[2] * x[3] * x[4]
    g[2] = x[1]^2 + x[2]^2 + x[3]^2 + x[4]^2
    return g
end

##------------------------ grad obj function ------------------------##

# grad of obj function ∇f(x)
grad_f = zeros(n)
# analytical grad of obj function
function eval_grad_f(x::Vector{Float64}, grad_f::Vector{Float64})
    grad_f[1] = x[1] * x[4] + x[4] * (x[1] + x[2] + x[3])
    grad_f[2] = x[1] * x[4]
    grad_f[3] = x[1] * x[4] + 1
    return grad_f[4] = x[1] * (x[1] + x[2] + x[3])
end

# now with automatic differentiation
∇f = x -> ForwardDiff.gradient(eval_f, x)

# check both are the same
eval_grad_f(x_0, grad_f)
@show grad_f
@show ∇f(x_0)

function eval_grad_f_ad(x::Vector{Float64}, grad_f::Vector{Float64})
    grad_f[:] = ForwardDiff.gradient(eval_f, x)[:]
end
@show eval_grad_f_ad(x_0, grad_f)

##--------------------- jac of constraints ∇g(x)^T --------------------##
println("jacobian of constraints")
g = zeros(m)
∇g = x -> ForwardDiff.jacobian(x->eval_g_ad(x, g), x)
jac_temp = ∇g(x_0)
sparsity = jac_temp .!= 0.0
I = ((i, j) for i = 1:m, j = 1:n if sparsity[i,j] != 0.0)
rows = Int32.(getindex.(I, 1)[:])
cols = Int32.(getindex.(I, 2)[:])
values = zeros(length(rows))

# analytical version
function eval_jac_g(
    x::Vector{Float64},
    rows::Vector{Int32},
    cols::Vector{Int32},
    values::Union{Nothing,Vector{Float64}},
)
    if values === nothing
    # return the structure of the jacobian,
    # Sparse Triplet Matrix Format
    # i.e. the rows and cols of the non-zero entries
    # index style=FORTRAN STYLE (1-indexed)
        # Constraint (row) 1
        rows[1] = 1
        cols[1] = 1
        rows[2] = 1
        cols[2] = 2
        rows[3] = 1
        cols[3] = 3
        rows[4] = 1
        cols[4] = 4
        # Constraint (row) 2
        rows[5] = 2
        cols[5] = 1
        rows[6] = 2
        cols[6] = 2
        rows[7] = 2
        cols[7] = 3
        rows[8] = 2
        cols[8] = 4
    else
    # return the values of the jacobian of the constraints
        # Constraint (row) 1
        values[1] = x[2] * x[3] * x[4]  # 1,1
        values[2] = x[1] * x[3] * x[4]  # 1,2
        values[3] = x[1] * x[2] * x[4]  # 1,3
        values[4] = x[1] * x[2] * x[3]  # 1,4
        # Constraint (row) 2
        values[5] = 2 * x[1]  # 2,1
        values[6] = 2 * x[2]  # 2,2
        values[7] = 2 * x[3]  # 2,3
        values[8] = 2 * x[4]  # 2,4
    end
    @show values
    return nothing
end
println("analytical")
eval_jac_g(x_0, rows, cols, nothing)
@show rows; @show cols
eval_jac_g(x_0, rows, cols, values)
@show values
display(reshape(values, m, n))

println("\nAD")

function jac_structure!(
    m, n,
    rows::AbstractVector{<:Integer},
    cols::AbstractVector{<:Integer},
  )
    I = ((i, j) for i = 1:m, j = 1:n)
    rows .= getindex.(I, 1)[:]
    cols .= getindex.(I, 2)[:]
    return rows, cols
end

function jac_structure_sparse!(
    x, m, n,
    rows::AbstractVector{<:Integer},
    cols::AbstractVector{<:Integer},
  )
    jac_temp =  ForwardDiff.jacobian(x->eval_g_ad(x, g), x)
    sparsity = jac_temp .!= 0.0
    I = ((i, j) for i = 1:m, j = 1:n if sparsity[i,j] != 0.0)
    rows .= getindex.(I, 1)[:]
    cols .= getindex.(I, 2)[:]
    return rows, cols
end

∇g = x -> ForwardDiff.jacobian(x->eval_g_ad(x, g), x)

function eval_jac_g_ad(x, rows, cols, values)

    if values === nothing
        jac_structure_sparse!(x_0, m, n, rows, cols)
        # jac_structure!(m, n, rows, cols)
    else

        values[:] = ForwardDiff.jacobian(x->eval_g_ad(x, g), x)'[:]
    end
    return nothing
end

#jac_structure!(m, n, rows, cols)
@show jac_structure_sparse!(x_0, m, n, rows, cols)
@show rows; @show cols

eval_jac_g_ad(x_0, rows, cols, nothing)
eval_jac_g_ad(x_0, rows, cols, values)

##------------------------ Hessian of Langrangian  ------------------------##
# L = f(x) + g(x)^T * λ
# ∇L = σ_f  ∇^2 f(x_k) + Σ^m_i λ_i * ∇^2 g_i(x_k)
println("Hessian of Langrangian")
values = ones(10)

function eval_h(
    x::Vector{Float64},
    rows::Vector{Int32},
    cols::Vector{Int32},
    obj_factor::Float64,
    lambda::Vector{Float64},
    values::Union{Nothing,Vector{Float64}},
)
    if values === nothing
        # Symmetric matrix, fill the lower left triangle only
        idx = 1
        for row in 1:4
            for col in 1:row
                rows[idx] = row
                cols[idx] = col
                idx += 1
            end
        end
    else
        # Again, only lower left triangle
        # Objective
        values[1] = obj_factor * (2 * x[4])  # 1,1
        values[2] = obj_factor * (x[4])  # 2,1
        values[3] = 0                      # 2,2
        values[4] = obj_factor * (x[4])  # 3,1
        values[5] = 0                      # 3,2
        values[6] = 0                      # 3,3
        values[7] = obj_factor * (2 * x[1] + x[2] + x[3])  # 4,1
        values[8] = obj_factor * (x[1])  # 4,2
        values[9] = obj_factor * (x[1])  # 4,3
        values[10] = 0                     # 4,4

        # First constraint
        values[2] += lambda[1] * (x[3] * x[4])  # 2,1
        values[4] += lambda[1] * (x[2] * x[4])  # 3,1
        values[5] += lambda[1] * (x[1] * x[4])  # 3,2
        values[7] += lambda[1] * (x[2] * x[3])  # 4,1
        values[8] += lambda[1] * (x[1] * x[3])  # 4,2
        values[9] += lambda[1] * (x[1] * x[2])  # 4,3

        # Second constraint
        values[1] += lambda[2] * 2  # 1,1
        values[3] += lambda[2] * 2  # 2,2
        values[6] += lambda[2] * 2  # 3,3
        values[10] += lambda[2] * 2  # 4,4
    end
    return
end

println("analytical")
eval_h(x_0, Int32.(ones(10)), Int32.(ones(10)), 1., ones(2), nothing)
# Hessian of constraints
eval_h(x_0, Int32.(ones(10)), Int32.(ones(10)), 0., ones(2), values)
@show values

function L(x, λ, σ)
    λ = typeof(x)(λ)
    return σ * eval_f(x) + eval_g_ad(x,g)'*λ
end

λ = ones(2)
σ = 0.0
L(x_0, λ, σ)
∇H = x -> ForwardDiff.hessian(x->L(x, λ, σ), x)
sparsity = LowerTriangular(∇H(x_0)) .!= 0.0
I = ((i, j) for i = 1:size(sparsity,1), j = 1:size(sparsity,2) if sparsity[i,j] != 0.0)
rows_hess = Int32.(getindex.(I, 1)[:])
cols_hess = Int32.(getindex.(I, 2)[:])
values_hess = zeros(length(rows_hess))

function hess_structure_sparse!(x, rows, cols)
    ∇H = x -> ForwardDiff.hessian(x->L(x, λ, 1), x)
    sparsity = LowerTriangular(∇H(x_0)) .!= 0.0
    I = ((i, j) for i = 1:size(sparsity,1), j = 1:size(sparsity,2) if sparsity[i,j] != 0.0)
    rows .= Int32.(getindex.(I, 1)[:])
    cols .= Int32.(getindex.(I, 2)[:])
    return rows, cols
end

function eval_h_ad(
    x::Vector{Float64},
    rows,
    cols,
    σ::Float64,
    λ::Vector{Float64},
    values::Union{Nothing,Vector{Float64}},
)
    if values === nothing
        hess_structure_sparse!(x, rows, cols)
    else
        hess = ForwardDiff.hessian(x->L(x, λ, σ), x)
        values[:] = hess[tril(trues(size(hess)))]
    end
end
println("automatic differentiation")
eval_h_ad(x_0, rows_hess, cols_hess, σ, λ, values_hess)
@show rows_hess; @show cols_hess
eval_h_ad(x_0, rows_hess, cols_hess, σ, λ, values_hess)
@show values_hess

##------------------------ Solve problem analytical ------------------------##

n = 4
x_L = [1.0, 1.0, 1.0, 1.0]
x_U = [5.0, 5.0, 5.0, 5.0]

m = 2
g_L = [25.0, 40.0]
g_U = [2.0e19, 40.0]

prob = Ipopt.CreateIpoptProblem(
    n,
    x_L,
    x_U,
    m,
    g_L,
    g_U,
    8,
    10,
    eval_f,
    eval_g,
    eval_grad_f,
    eval_jac_g,
    eval_h,
)

prob.x = [1.0, 5.0, 5.0, 1.0]
solvestat = Ipopt.IpoptSolve(prob)
@show prob.x
##------------------------ solve with ad ------------------------##

n = 4
x_L = [1.0, 1.0, 1.0, 1.0]
x_U = [5.0, 5.0, 5.0, 5.0]

m = 2
g_L = [25.0, 40.0]
g_U = [2.0e19, 40.0]

g = zeros(m)
∇g = x -> ForwardDiff.jacobian(x->eval_g_ad(x, g), x)
jac_temp = ∇g(x_0)
sparsity = jac_temp .!= 0.0
I = ((i, j) for i = 1:m, j = 1:n if sparsity[i,j] != 0.0)
rows = Int32.(getindex.(I, 1)[:])
cols = Int32.(getindex.(I, 2)[:])
values = zeros(length(rows))
rows = nothing
cols = nothing
values = nothing

prob = Ipopt.CreateIpoptProblem(
    n,
    x_L,
    x_U,
    m,
    g_L,
    g_U,
    8,
    10,
    eval_f,
    eval_g,
    eval_grad_f_ad,
    eval_jac_g_ad,
    eval_h_ad,
    # function h() return false end
)
# Ipopt.AddIpoptStrOption(prob, "hessian_approximation", "limited-memory")
# Ipopt.AddIpoptStrOption(prob, "check_derivatives_for_naninf", "yes")
# Ipopt.AddIpoptIntOption(prob, "file_print_level", 10)
# Ipopt.AddIpoptIntOption(prob, "output_file", "ipopt.out")
# Ipopt.OpenIpoptOutputFile(prob, "blah.txt", 5)

prob.x = [1.0, 5.0, 5.0, 1.0]
solvestat = Ipopt.IpoptSolve(prob)
@show prob.x


struct LinearKF
    xk::Vector
	xp::Vector
	P::AbstractMatrix
	K::AbstractMatrix
	S::AbstractMatrix
	y::Vector
	filter::SquareRootFilter
end

"""
	sol = linear_kalman(
		x, x₀, P, F, H, Q, R;
		B=zeros(length(x), length(x)),
		u=zeros(length(x)),
		w=zeros(length(x)),
		v=zeros(length(x)),
		sqrt_filter=CholeskySqrt(),
		)

Performs the simple Kalman filter algorithm for the estimation of a state x.

x:   Vector with one measurement (observation) of m = length(x) variables
x₀:  A priori estimation of the variables
P:   Initial predicted process covariance matrix
F:	 Transition matrix (linear state model)
H:	 Measurement matrix (linear measurement model)
Q:   Process noise covariance matrix
R:   Measurement (observable) covariance matrix

B:	 Control-input matrix (linear state model)
u:	 Control vector (linear state model)
w:	 Process noise vector, w ~ N(0,Q)
v:	 Measurement (observable) noise vector, v ~ N(0,R)

sqrt_filter: Square root filtering technique for the covariance matrix.
			 Accepted values are: CholeskyModSqrt() (default), SVDSqrt(), CholeskySqrt()

sol: LinearKalman type
    xk:     Kalman filter state estimation
    xp:     Predicted state estimate
    P:      Process covariance matrix a posteriori
    K:      Kalman gain
	S:      Innovation (or pre-fit residual) covariance
	y:      Innovation or measurement pre-fit residual
	filter: Type of SquareRootFilter selected

"""
function linear_kalman(
	x::Vector,
    x₀::Vector,
    P::AbstractMatrix,
	F::AbstractMatrix,
	H::AbstractMatrix,
	Q::AbstractMatrix,
	R::AbstractMatrix;
	B::AbstractMatrix=zeros(length(x), length(x)),
	u::Union{Vector, Real}=zeros(length(x)),
	w::Vector=zeros(length(x)),
	v::Vector=zeros(length(x)),
	sqrt_filter::SquareRootFilter=CholeskyModSqrt(),
	)

	m = length(x)

	## Predict process
	# State estimate propagation (Predicted state, a priori)
	x̂ = _linear_state_model(x₀, F, B, u, w)
	# Error covariance propagation (Predicted process covariance matrix, a priori)
	P_ = _process_covariance_priori(P, F, Q)
	# Innovation (or pre-fit residual) covariance
	PHt = P_ * H'
	S = _innovation_covariance(P_, PHt, H, R)
	# Kalman gain
	K = _kalman_gain(P_, PHt, S)

	## Update process
	# New observation (innovation process)
    y = _linear_observation_model(x, H, v)
	# Innovation or measurement pre-fit residual
	y = muladd(-H, x̂, y)
	# State estimate update (Current state, a posteriori)
	xₖ = muladd(K, y, x̂)
	# Error covariance update (Process covariance matrix, a posteriori)
	P_ = _process_covariance_posteriori(P_, H, K, m)

	return LinearKF(vec(xₖ), vec(x̂), P_, K, S, vec(y), sqrt_filter)
end

function _linear_state_model(
	x::Vector, F::AbstractMatrix, B::AbstractMatrix, u::Union{Vector, Real}, w::Vector,
	)
	return vec(F*x .+ B .* u .+ w)
end

function _linear_observation_model(x::Vector, H::AbstractMatrix, v::Vector)
	return vec(muladd(H, x, v))
end

function _process_covariance_priori(P::AbstractMatrix, F::AbstractMatrix, Q::AbstractMatrix)
	return _return_type(P, F*P*F' .+ Q)
end

function _process_covariance_posteriori(
	P::AbstractMatrix, H::AbstractMatrix, K::AbstractMatrix, m::Int64,
	)
	return _return_type(P, (eye(m) .- K*H) *P)
end

function _kalman_gain(P::AbstractMatrix, PHt::AbstractMatrix, S::AbstractMatrix)
	F = qr(S)
	return _return_type(P, F.R \ (F.Q' * PHt))
end

function _innovation_covariance(
	P::AbstractMatrix, PHt::AbstractMatrix, H::AbstractMatrix, R::AbstractMatrix,
	)
	return _return_type(P, H*PHt .+ R)
end


using Polynomials
"""
    _getpolylagrange(k::Int64, order::Int64, origin::Int64)

Function that return the k-th Lagrange Polynomial of a certain order. Coefficients are rational then the return is exact. The polynomial is equal to :
``\\prod_{i=0,\\ i \\neq k}^{order} \\frac{x - i - origin}{k - i}``

# Arguments
- `k::Int64` : number of the Polynomial, `k` must be between `0` and `order` (`0<= k <= order`).
- `order::Int64` : order of the polynomial.
- `origin::Int64` : origin of the first indice.

# Returns
- `Polynomial{Rational{BigInt}}` : the k-th Lagrange polynomial of order `order`

# Throws
- `DommaineError` : when `0 <= k <= order` is `false` or when N ∉ {BInt64, BigInt}
"""
function _getpolylagrange(k::Int, order::Int, origin::Int) where {N<:Integer}
    0 <= k <= order || throw(DomainError("the constant 0 <= k <= order is false"))
    # the computed is made with big rational
    result = Polynomials.Polynomial([big(1 // 1)])
    for l = 0:order
        if l != k
            result *= Polynomials.Polynomial([-(l + origin), 1 // 1] .// (k - l))
        end
    end
    return result
end

"""
    Lagrange{T, edge, order, N} <: AbstractInterpolation{T, edge, order}
    Lagrange(order, T::DataType=Float64; edge::EdgeType=CircEdge)

Type containing Lagrange Polynomials coefficients for Lagrange interpolation

# Type parameters
- `T` : the type of data that is interpolate
- `edge::EdgeType` : type of edge traitment
- `order::Int`: order of lagrange interpolation

# Implementation :
- `tabfct::Vector{Polynomial{T}}` : vector of all lagrange polynomial, per example the k-th Lagrange polynomial for the designed order is tabfct[k+1]

# Arguments :
- `order::Int` : the order of interpolation
- `[T::DataType=Float64]` : The type values to interpolate

# Keywords arguments :
- `edge::EdgeType=CircEdge` : type of edge traitment

"""
struct Lagrange{T,edge,order} <: AbstractInterpolation{T,edge,order}
    tabfct::Vector{Polynomial{T}}
    function Lagrange(order::Int, T::DataType = Float64; edge::EdgeType = CircEdge)
        origin = -div(order, 2)
        tabfct_rat = collect([_getpolylagrange(i, order, origin) for i = 0:order])
        new{T,edge,order}(convert.(Polynomial{T}, tabfct_rat))
    end
end

function _c(k,n)
    p = integrate(_getpolylagrange(k,n,0))
    return p(0)-p(-1)
end
struct ABcoef
    tab::Array{Rational{Int}}
    function ABcoef(ordermax::Int)
        tab = zeros(Rational{Int}, ordermax, ordermax)
        for j=1:ordermax, i=1:j
            tab[i,j] = _c(i-1,j-1)
        end
        return new(tab)
    end
end
c(st::ABcoef, k, n)=st.tab[k,n]


"""

The sum of the primes below 10 is 2 + 3 + 5 + 7 = 17.

Find the sum of all the primes below two million.
"""

function primesum(n::Int)
    isprime = trues(n)
    isprime[1] = false

    for i in 2:Int(trunc(√n))
        if isprime[i]
            for j in i^2:i:n
                isprime[j] = false
            end
        end
    end
    return sum(filter(x->isprime[x],1:n))
end

@assert primesum(10) == 17
@show primesum(2000000)


# [1](https://projecteuler.net/problem=1)

# \sum_i ia + \sum_j jb  \text{ ,where } i \neq nb , j \neq ma

@time s = sum([ i for i in filter(x -> (x%3==0) || (x%5==0), 1:999) ])
println(s)
# slow
# 233168

# equivalent to \sum_i ia + \sum_j jb - \sum_k kab
# where kab is the repeated numbers among ia and jb.

@time s = sum([ 3:3:999; 5:5:999; -(15:15:999) ])
println(s)
# fast
# 233168


using Random, Distributions, Plots; pyplot()
Random.seed!(1)

Z() = sqrt(-2*log(rand()))*cos(2*pi*rand())
xGrid = -4:0.01:4

histogram([Z() for _ in 1:10^6], bins=50,
		normed=true, label="MC estimate")
plot!(xGrid, pdf.(Normal(),xGrid),
	 c=:red, lw=4, label="PDF",
	 xlims=(-4,4), ylims=(0,0.5), xlabel="x", ylabel="f(x)")


"""
    poweriter(A,numiter)

Perform `numiter` power iterations with the matrix `A`, starting
from a random vector. Returns a vector of eigenvalue estimates
and the final eigenvector approximation.
"""
function poweriter(A,numiter)
    n = size(A,1)
    x = normalize(randn(n),Inf)
    β = zeros(numiter)
    for k in 1:numiter
        y = A*x
        m = argmax(abs.(y))
        β[k] = y[m]/x[m]
        x = y/y[m]
    end
    return β,x
end

"""
    inviter(A,s,numiter)

Perform `numiter` inverse iterations with the matrix `A` and shift
`s`, starting from a random vector. Returns a vector of
eigenvalue estimates and the final eigenvector approximation.
"""
function inviter(A,s,numiter)
    n = size(A,1)
    x = normalize(randn(n),Inf)
    β = zeros(numiter)
    fact = lu(A - s*I)
    for k in 1:numiter
        y = fact\x
        normy,m = findmax(abs.(y))
        β[k] = x[m]/y[m] + s
        x = y/y[m]
    end
    return β,x
end

"""
    arnoldi(A,u,m)

Perform the Arnoldi iteration for `A` starting with vector `u`, out
to the Krylov subspace of degree `m`. Returns the orthonormal basis
(`m`+1 columns) and the upper Hessenberg `H` of size `m`+1 by `m`.
"""
function arnoldi(A,u,m)
    n = length(u)
    Q = zeros(n,m+1)
    H = zeros(m+1,m)
    Q[:,1] = u/norm(u)
    for j in 1:m
        # Find the new direction that extends the Krylov subspace.
        v = A*Q[:,j]
        # Remove the projections onto the previous vectors.
        for i in 1:j
            H[i,j] = dot(Q[:,i],v)
            v -= H[i,j]*Q[:,i]
        end
        # Normalize and store the new basis vector.
        H[j+1,j] = norm(v)
        Q[:,j+1] = v/H[j+1,j]
    end
    return Q,H
end

"""
    gmres(A,b,m)

Do `m` iterations of GMRES for the linear system `A`*x=`b`. Returns
the final solution estimate x and a vector with the history of
residual norms. (This function is for demo only, not practical use.)
"""
function gmres(A,b,m)
    n = length(b)
    Q = zeros(n,m+1)
    Q[:,1] = b/norm(b)
    H = zeros(m+1,m)

    # Initial solution is zero.
    x = 0
    residual = [norm(b);zeros(m)]

    for j in 1:m
        # Next step of Arnoldi iteration.
        v = A*Q[:,j]
        for i in 1:j
            H[i,j] = dot(Q[:,i],v)
            v -= H[i,j]*Q[:,i]
        end
        H[j+1,j] = norm(v)
        Q[:,j+1] = v/H[j+1,j]

        # Solve the minimum residual problem.
        r = [norm(b); zeros(j)]
        z = H[1:j+1,1:j] \ r
        x = Q[:,1:j]*z
        residual[j+1] = norm( A*x - b )
    end
    return x,residual
end

### These are not part of the text proper, but help to construct
### interesting examples.

"""
    sprandsym(n,density,λ)
    sprandsym(n,density,rcond)

Construct a randomized `n` by `n` symmetric sparse matrix of approximate
density `density`. For vector `λ`, the matrix has eigenvalues as
prescribed by λ. For scalar `rcond`, the matrix has condition number
equal to 1/`rcond`.
"""
function sprandsym(n,density,rcond::Number)
    λ = [ rcond^(i/(n-1)) for i in 0:n-1 ]
    sprandsym(n,density,λ)
end

function sprandsym(n,density,λ::Vector)
    # Apply a random Jacobi rotation similarity transformation.
    function randjr(A)
        theta = 2π*rand()
        c,s = cos(theta),sin(theta)
        i = j = 0
        while i==j
          i,j = rand(1:n,2)
        end
        A[[i,j],:] = [c s;-s c]*A[[i,j],:]
        A[:,[i,j]] = A[:,[i,j]]*[c -s;s c]
        return A
    end

    targetnz = ceil(min(0.98,density)*n^2)
    A = spdiagm(λ)

    while nnz(A) < targetnz
        A = randjr(A)
    end
    return A
end

"""
    poisson(n)

Construct the finite difference Laplacian matrix for an `n` by `n`
interior lattice.
"""
function poisson(n)
    D = spdiagm(-1=>fill(-1,n-1),0=>fill(2,n),1=>fill(-1,n-1)) * (n+1)^2/pi^2
    return kron(D,I(n)) + kron(I(n),D)
end


# Example: Compartmental models in epidemiology
#=
- https://github.com/epirecipes/sir-julia/blob/master/markdown/function_map/function_map.md
- https://en.wikipedia.org/wiki/Compartmental_models_in_epidemiology#Deterministic_versus_stochastic_epidemic_models
=#
using ModelingToolkit, Test

@inline function rate_to_proportion(r,t)
    1-exp(-r*t)
end;

# Independent and dependent variables and parameters
@parameters t c nsteps δt β γ
D = Difference(t; dt=0.1)
@variables S(t) I(t) R(t) next_S(t) next_I(t) next_R(t)

infection = rate_to_proportion(β*c*I/(S+I+R),δt)*S
recovery = rate_to_proportion(γ,δt)*I

# Equations
eqs = [D(S) ~ S-infection,
       D(I) ~ I+infection-recovery,
       D(R) ~ R+recovery]

# System
@named sys = DiscreteSystem(eqs,t,[S,I,R],[c,nsteps,δt,β,γ]; controls = [β, γ])

# Problem
u0 = [S => 990.0, I => 10.0, R => 0.0]
p = [β => 0.05, c => 10.0, γ => 0.25, δt => 0.1, nsteps => 400]
tspan = (0.0,ModelingToolkit.value(substitute(nsteps,p))) # value function (from Symbolics) is used to convert a Num to Float64
prob_map = DiscreteProblem(sys,u0,tspan,p)

# Solution
using OrdinaryDiffEq
sol_map = solve(prob_map,FunctionMap());

# Direct Implementation

function sir_map!(u_diff,u,p,t)
    (S,I,R) = u
    (β,c,γ,δt) = p
    N = S+I+R
    infection = rate_to_proportion(β*c*I/N,δt)*S
    recovery = rate_to_proportion(γ,δt)*I
    @inbounds begin
        u_diff[1] = S-infection
        u_diff[2] = I+infection-recovery
        u_diff[3] = R+recovery
    end
    nothing
end;
u0 = [990.0,10.0,0.0];
p = [0.05,10.0,0.25,0.1];
prob_map = DiscreteProblem(sir_map!,u0,tspan,p);
sol_map2 = solve(prob_map,FunctionMap());

@test Array(sol_map) ≈ Array(sol_map2)

# Delayed difference equation
@parameters t
@variables x(..) y(..) z(t)
D1 = Difference(t; dt=1.5)
D2 = Difference(t; dt=2)

@test ModelingToolkit.is_delay_var(Symbolics.value(t), Symbolics.value(x(t-2)))
@test ModelingToolkit.is_delay_var(Symbolics.value(t), Symbolics.value(y(t-1)))
@test !ModelingToolkit.is_delay_var(Symbolics.value(t), Symbolics.value(z))
@test_throws ErrorException ModelingToolkit.get_delay_val(Symbolics.value(t), Symbolics.arguments(Symbolics.value(x(t+2)))[1])
@test_throws ErrorException z(t)

# Equations
eqs = [
    D1(x(t)) ~ 0.4x(t) + 0.3x(t-1.5) + 0.1x(t-3),
    D2(y(t)) ~ 0.3y(t) + 0.7y(t-2) + 0.1z,
]

# System
@named sys = DiscreteSystem(eqs,t,[x(t),x(t-1.5),x(t-3),y(t),y(t-2),z],[])

eqs2, max_delay = ModelingToolkit.linearize_eqs(sys; return_max_delay=true)

@test max_delay[Symbolics.operation(Symbolics.value(x(t)))] ≈ 3
@test max_delay[Symbolics.operation(Symbolics.value(y(t)))] ≈ 2

linearized_eqs = [
    eqs
    x(t - 3.0) ~ x(t - 1.5)
    x(t - 1.5) ~ x(t)
    y(t - 2.0) ~ y(t)
]
@test all(eqs2 .== linearized_eqs)


function circlecenter(coords1, coords2, radius)
    # two circle intersection -> https://math.stackexchange.com/a/1367732
    dist = norm(coords1-coords2)
    midpoint = (coords1+coords2)/2
    normaldist = sqrt(radius^2 - dist^2/4)
    normalvec = reverse((coords1 - coords2)) .* [1, -1]
    normalvec /= norm(normalvec)
    center = midpoint .+ sign(radius) .* normalvec .* normaldist

    return center
end

function circlesection(coords1, coords2, center, u)
    leg1 = coords1 .- center
    leg2 = coords2 .- center

    γ = angle(vec(leg1), vec(leg2))

    x(au) = leg1[1] * cos(au*γ) - leg1[2] * sin(au*γ) + center[1]
    y(au) = leg1[1] * sin(au*γ) + leg1[2] * cos(au*γ) + center[2]

    return (x.(u), y.(u))
end

function boundaryfun(boundary::StraightBoundary, vertices, inverse_direction=false)
    coords1 = vertices[boundary.vertice_start]
    coords2 = vertices[boundary.vertice_end]

    connection = coords2 - coords1

    if inverse_direction
        return coords2, u -> coords2 - connection .* u
    else
        return coords1, u -> coords1 + connection .* u
    end
end

function boundaryfun(boundary::CircularBoundary, vertices, inverse_direction=false)
    coords1 = vertices[boundary.vertice_start]
    coords2 = vertices[boundary.vertice_end]

    center = circlecenter(coords1, coords2, boundary.radius)

    leg1 = coords1 .- center
    leg2 = coords2 .- center

    γ = angle(vec(leg1), vec(leg2))

    T(u) = [cos(u*γ) -sin(u*γ); sin(u*γ) cos(u*γ)]

    if inverse_direction
        return coords2, u -> T(1-u) * leg1 + center
    else
        return coords1, u -> T(u) * leg1 + center
    end
end


# # Non-linear effects

using Unfold
using CairoMakie
using DataFrames
using Random
using Colors

# ## Generating a non-linea signal
# We start with generating
rng = MersenneTwister(2) # make repeatable
n = 20 # datapoints
evts = DataFrame(:x=>rand(rng,n))
signal = -(3*(evts.x .-0.5)).^2 .+ 0.5 .* rand(rng,n)

plot(evts.x,signal)
#
# Looks perfectly non-linear - great!
#
# # Compare linear & non-linear fit
# First we have to reshape it to fit to Unfold format, a 3d array,  1 channel x 1 timepoint x 20 datapoints
signal = reshape(signal,length(signal),1,1)
signal = permutedims(signal,[3,2,1])
size(signal)

# Next we define three different models. **linear** **3 splines** and **10 splines**
# note the different formulas, one `x` the other `spl(x,3)`
design_linear = Dict(Any=>	(@formula(0~1+x),[0]))
design_spl3   = Dict(Any=>	(@formula(0~1+spl(x,3)),[0]))
design_spl10  = Dict(Any=>	(@formula(0~1+spl(x,10)),[0])) #hide

# and we fit the parameters
uf_linear = fit(UnfoldModel,design_linear,evts,signal);
uf_spl3 = fit(UnfoldModel,design_spl3,evts,signal);
uf_spl10 = fit(UnfoldModel,design_spl10,evts,signal); #hide

# extract the fitted values using Unfold.effects

p_linear = Unfold.effects(Dict(:x => range(0,stop=1,length=100)),uf_linear);
p_spl3 = Unfold.effects(Dict(:x => range(0,stop=1,length=100)),uf_spl3);
p_spl10 = Unfold.effects(Dict(:x => range(0,stop=1,length=100)),uf_spl10);
first(p_linear,5)

# And plot them
pl = plot(evts.x,signal[1,1,:])
	lines!(p_linear.x,p_linear.yhat)
	lines!(p_spl3.x,p_spl3.yhat)
	lines!(p_spl10.x,p_spl10.yhat)
pl

# What we can clearly see here, that the linear effect (blue) underfits the data, the `spl(x,10)` overfits it, but the `spl(x,3)` fits it perfectly.

# ## Looking under the hood
# Let's have a brief look how the splines manage what they are managing.
#
# The most important bit to understand is, that we are replacing `x` by a set of coefficients `spl(x)`.
# These new coefficients each tile the range of `x` (in our case from [0-1]) in overlapping areas, each which will be fit by one coefficient.
# Because the ranges are overlapping, we get a smooth function.
#
# Maybe this becomes clear after looking at such a basisfunction:

term_spl = Unfold.formula(uf_spl10).rhs.terms[2]

# This is the spline term.
typeof(term_spl)

# a special type generated in Unfold.jl
#
# it has a field .fun which is the spline function. We can evaluate it at a point
term_spl.fun([0.2])

# each column of this 1 row matrix is a coefficient for our regression model.
lines(disallowmissing(term_spl.fun([0.2]))[1,:])

# Note: We have to use disallowmissing, because our splines return a `missing` whenever we ask it to return a value outside it's defined range, e.g.:
term_spl.fun([-0.2])

# because it never has seen any data outside and can't extrapolate!

# Okay back to our main issue: Let's plot the whole basis set
basisSet = term_spl.fun(0.:0.01:1)
basisSet = disallowmissing(basisSet[.!any(ismissing.(basisSet),dims=2)[:,1],:]) # remove missings
ax = Axis(Figure()[1,1])
[lines!(ax,basisSet[:,k]) for k in 1:size(basisSet,2)]
current_figure()

# notice how we flipped the plot around, i.e. now on the x-axis we do not plot the coefficients, but the `x`-values
# Now each line is one basis-function of the spline.
#
# Unfold returns us one coefficient per basis-function
β=coef(uf_spl10)[1,1,:]
β = Float64.(disallowmissing(β))

# but because we used an intercept, we have to do some remodelling in the basisSet
X = hcat(ones(size(basisSet,1)), basisSet[:,1:5], basisSet[:,7:end])

# now we can weight the spline by the basisfunction
weighted = (β .*X')

# Plotting them makes for a nice looking plot!
ax = Axis(Figure()[1,1])
[lines!(weighted[k,:]) for k = 1:11]
current_figure()

# and sum them up
lines(sum(weighted,dims=1)[1,:])
plot!(X*β,color="gray") #(same as matrixproduct X*β directly!)
current_figure()

# And this is how you can think about splines


# Functions for area of various geometric shapes
# Contributed By:- [Ashwani Rathee](https://github.com/ashwani-rathee)

"""
    surfarea_cube(side)

Finds surface area of a cube

# Example

```julia
surfarea_cube(1)  # returns 6
surfarea_cube(3)  # returns 54
surfarea_cube(-1) # returns DomainError
```
"""
function surfarea_cube(side)
    if side < 0
        throw(
            DomainError(
                side,
                "surfarea_cube() only works for non-negative values",
            ),
        )
    end
    return 6(side^2)
end

"""
    surfarea_sphere(side)

Finds surface area of a sphere

# Example

```julia
surfarea_sphere(5)  # returns 314.1592653589793
surfarea_sphere(1)  # returns 12.566370614359172
surfarea_sphere(-1) # returns DomainError
```
"""
function surfarea_sphere(radius)
    if radius < 0
        throw(
            DomainError(
                radius,
                "surfarea_sphere() only works for non-negative values",
            ),
        )
    end
    return 4π * (radius^2)
end

"""
    area_rectangle(length, width)

Finds area of the rectangle

# Example

```julia
area_rectangle(10,20) # returns 200
area_rectangle(-1,-2) # returns DomainError
area_rectangle(1,-2)  # returns DomainError
area_rectangle(-1,2)  # returns DomainError
```
"""
function area_rectangle(length, width)
    if length < 0 || width < 0
        throw(
            DomainError("area_rectangle() only works for non-negative values"),
        )
    end
    return length * width
end

"""
    area_square(side)

Finds area of the area_square

# Example

```julia
area_square(10) # returns 100
area_square(-1) # returns DomainError
```
"""
function area_square(side)
    if side < 0
        throw(DomainError("area_square() only works for non-negative values"))
    end
    return side^2
end

"""
    area_triangle(base, height)

Finds area of the right angled triangle with base height

# Example

```julia
area_triangle(10,10) # returns 50.0
area_triangle(-1,-2) # returns DomainError
area_triangle(1,-2)  # returns DomainError
area_triangle(-1,2)  # returns DomainError
```
"""
function area_triangle(base, height)
    if base < 0 || height < 0
        throw(DomainError("area_triangle() only accepts non-negative values"))
    end
    return (base * height) / 2
end

"""
    area_heron_triangle(side1, side2, side3)

Finds area of a triangle using heron's formula

# Example

```julia
area_heron_triangle(5,12,13) # returns 30.0
area_heron_triangle(-1,-2,1) # returns DomainError
area_heron_triangle(1,-2,1)  # returns DomainError
area_heron_triangle(-1,2,1)  # returns DomainError
```
"""
function area_heron_triangle(side1, side2, side3)
    if side1 < 0 || side2 < 0 || side3 < 0
        throw(
            DomainError(
                "area_triangle_three_sides() only accepts non-negative values",
            ),
        )
    elseif side1 + side2 < side3 ||
           side1 + side3 < side2 ||
           side2 + side3 < side1
        throw(DomainError("Given three sides do not form a triangle"))
    end
    semi_perimeter = (side1 + side2 + side3) / 2
    area = sqrt(
        semi_perimeter *
        (semi_perimeter - side1) *
        (semi_perimeter - side2) *
        (semi_perimeter - side3),
    )
    return area
end

"""
    area_parallelogram(base, height)

Finds area of the parallelogram

# Example

```julia
area_parallelogram(10,20) # returns 200
area_parallelogram(-1,-2) # returns DomainError
area_parallelogram(1,-2)  # returns DomainError
area_parallelogram(-1,2)  # returns DomainError
```
"""
function area_parallelogram(base, height)
    if base < 0 || height < 0
        throw(
            DomainError(
                "area_parallelogram() only accepts non-negative values",
            ),
        )
    end
    return base * height
end

"""
area_trapezium(base1,base2,height)

Finds area of the traπzium

# Example

```julia
area_trapezium(10, 20, 30)  # returns 450.0
area_trapezium(-1, -2, -3)  # returns DomainError
area_trapezium(-1, 2, 3)    # returns DomainError
area_trapezium(1, -2, 3)    # returns DomainError
area_trapezium(1, 2, -3)    # returns DomainError
area_trapezium(-1, -2, 3)   # returns DomainError
area_trapezium(1, -2, -3)   # returns DomainError
area_trapezium(-1, 2, -3)   # returns DomainError
```
"""
function area_trapezium(base1, base2, height)
    if base1 < 0 || base2 < 0 || height < 0
        throw(DomainError("area_trapezium() only accepts non-negative values"))
    end
    return 1 / 2 * (base1 + base2) * height
end

"""
    area_circle(radius)

Finds area of the circle

# Example

```julia
area_circle(20) # returns 1256.6370614359173
area_circle(-1) # returns DomainError
```
"""
function area_circle(radius)
    if radius < 0
        throw(DomainError("area_circle() only accepts non-negative values"))
    end
    return π * radius^2
end

"""
    area_ellipse(radius_x, radius_y)

Finds area of the ellipse

# Example

```julia
area_ellipse(10, 10) # returns 314.1592653589793
area_ellipse(10, 20) # returns 628.3185307179587
area_ellipse(1, -2) # returns DomainError
area_ellipse(-1, 2) # returns DomainError
area_ellipse(-1, -2) # returns DomainError
```
"""
function area_ellipse(radius_x, radius_y)
    if radius_x < 0 || radius_y < 0
        throw(DomainError("area_ellipse() only accepts non-negative values"))
    end
    return π * radius_x * radius_y
end

"""
    area_rhombus(diagonal_1, diagonal_2)

Finds area of the rhombus

# Example

```julia
area_rhombus(10, 20) # returns 100.0
area_rhombus(-1,-2) # returns DomainError
area_rhombus(1,-2)  # returns DomainError
area_rhombus(-1,2)  # returns DomainError
```
"""
function area_rhombus(diagonal_1, diagonal_2)
    if diagonal_1 < 0 || diagonal_2 < 0
        throw(DomainError("area_rhombus() only accepts non-negative values"))
    end
    return 1 / 2 * diagonal_1 * diagonal_2
end

"""
    trapazoidal_area(f, x_start, x_end, steps)

Approximates the area under the curve using the trapezoidal rule
Arguments:
 - f: function for the
 - x_start: starting value for x
 - x_end: ending value for x
 - steps: steps taken while integrating.
"""
function trapazoidal_area(f, x_start, x_end, steps)
    x1 = x_start
    fx1 = f(x_start)
    area = 0.0

    for _ in 1:steps
        x2 = ((x_end - x_start) / steps) + x1
        fx2 = f(x2)
        area += abs(fx2 + fx1) * (x2 - x1) / 2
        x1 = x2
        fx1 = fx2
    end

    return area
end

"""
    area_regular_polygon(sides, side_len)

Finds area of any regular Polygon

# Example

```julia
area_regular_polygon(1, 5)  # returns DomainError
area_regular_polygon(3, 5)  # returns 10.825317547305486
area_regular_polygon(7, 15)  # returns 817.6302999003576
area_regular_polygon(-1, 4) # returns DomainError
area_regular_polygon(4, -3) # returns DomainError
area_regular_polygon(-12, -4) # returns DomainError
```
"""
function area_regular_polygon(sides, side_len)
    sides < 3 && throw(DomainError("A polygon should have atleast 3 sides"))
    (sides < 0 || side_len < 0) && throw(
        DomainError(
            "area_regular_polygon() only works for non-negative values",
        ),
    )
    return sides * side_len^2 * cot(pi / sides) / 4
end

"""
    area_polygon(V)

Finds area of any Polygon given by continuous sequence of vertex coordinates
Arguments:
- coords: x,y co-ordinates of the vertices
    [Vector of Tuples / Matrix with 2 rows or 2 columns]

# Example

```julia
area_polygon([(0, 0), (100, 0), (0, 100)])  # returns 5000.0
area_polygon([0 0;100 0;100 100;0 100])  # returns 10000.0
area_polygon([(6, 4.5), (5, 4.5), (4.5, 5.5), (5, 6.5)]) # returns 1.5
area_polygon([0 0;100 0]) # returns DomainError
area_polygon([(6, 4.63), (5, 4.63)]) # returns DomainError
```
"""
function area_polygon(coords::Matrix{T}) where {T<:Real}
    r, c = size(coords)
    (~((c == 2 && r > 2) || (r == 2 && c > 2))) &&
        throw(DomainError("A polygon should have at least 3 vertices"))

    reshape(coords, (:, 2))
    if ~(coords[1, :] == coords[end, :])
        coords = vcat(coords, coords[1, :]') #to make a loop of the vertices
    end
    return @views abs(sum(coords[1:end-1, :] .* (-coords[2:end, 2:-1:1]))) / 2
end

function area_polygon(coords::Vector{Tuple{T,N}}) where {T,N<:Real}
    length(coords) < 3 &&
        throw(DomainError("A polygon should have at least 3 vertices"))
    if ~(first(coords) == last(coords))
        push!(coords, first(coords)) #to make a loop of the vertices
    end
    res = 0
    for i in 1:length(coords)-1
        res =
            res + coords[i][1] * coords[i+1][2] - coords[i+1][1] * coords[i][2]
    end
    return abs(res) / 2
end


using Distributions, Plots; pyplot()

function anovaFStat(allData)
    xBarArray = mean.(allData)
    nArray = length.(allData)
    xBarTotal = mean(vcat(allData...))
    L = length(nArray)

    ssBetween = sum( [nArray[i]*(xBarArray[i] - xBarTotal)^2 for i in 1:L] )
    ssWithin = sum([sum([(ob - xBarArray[i])^2 for ob in allData[i]])
				for i in 1:L])
    return (ssBetween/(L-1))/(ssWithin/(sum(nArray)-L))
end

case1 = [13.4, 13.4, 13.4, 13.4, 13.4]
case2 = [12.7, 11.8, 13.4, 12.7, 12.9]
stdDevs = [2, 2, 2, 2, 2]
numObs = [24, 15, 13, 23, 9]
L = length(case1)

N = 10^5

mcFstatsH0 = Array{Float64}(undef, N)
for i in 1:N
    mcFstatsH0[i] = anovaFStat([ rand(Normal(case1[j],stdDevs[j]),numObs[j])
		for j in 1:L ])
end

mcFstatsH1 = Array{Float64}(undef, N)
for i in 1:N
    mcFstatsH1[i] = anovaFStat([ rand(Normal(case2[j],stdDevs[j]),numObs[j])
		for j in 1:L ])
end

stephist(mcFstatsH0, bins=100,
	c=:blue, normed=true, label="Equal group means case")
stephist!(mcFstatsH1, bins=100,
	c=:red, normed=true, label="Unequal group means case")

dfBetween = L - 1
dfError = sum(numObs) - 1
xGrid = 0:0.01:10
plot!(xGrid, pdf.(FDist(dfBetween, dfError),xGrid),
            c=:black, label="F-statistic analytic")
critVal = quantile(FDist(dfBetween, dfError),0.95)
plot!([critVal, critVal],[0,0.8],
	c=:black, ls=:dash, label="Critical value boundary",
	xlims=(0,10), ylims=(0,0.8), xlabel="F-value", ylabel="Density")


# # Example 7
# This example is from the book Princeton Handbook of Test Problems in Local and Global Optimization
# Dempe, Chapter 9.3.2 -parg 223 [url](https://www.springer.com/gp/book/9780792358015)

# Here, only the second level is described

# Model of the problem
# First level
# ```math
# \min 2x_1+2x_2-3y_1-3y_2-60,\\
# \notag s.t.\\
# x_1 + x_2 + y_1 -2y_2 -40\leq 0,\\
# 0 \leq x_i \leq 50, \forall i \in I,\\
# -10 \leq y_j \leq 20, \forall j \in J,\\
# ```
# Second level
# ```math
# \min (-x_1 + y_1 + 40)^2 + (-x_2 + y_2 + 20)^2,\\
# \notag s.t.\\
# - x_i + 2y_j <= -10,\forall (i,j) \in \{(i,j)|i\in I, j\in J, i=j\},\\
# -10 \leq y_j \leq 20, \forall j \in J.\\
# ```

using BilevelJuMP
using Ipopt
using JuMP
using Test

model = BilevelModel(Ipopt.Optimizer, mode = BilevelJuMP.ProductMode(1e-9))

# First we need to create all of the variables in the upper and lower problems:

# Upper level variables
@variable(Upper(model), x[i=1:2], start = 0)

# Lower level variables
@variable(Lower(model), y[i=1:2], start = -10)

# Then we can add the objective and constraints of the upper problem:

# Upper level objecive function
@objective(Upper(model), Min,2x[1] + 2x[2] -3y[1] - 3y[2] -60)

# Upper level constraints
@constraint(Upper(model), x[1] + x[2] + y[1] -2y[2] -40 <= 0)
@constraint(Upper(model), [i=1:2], x[i] >= 0)
@constraint(Upper(model), [i=1:2], x[i] <= 50)
@constraint(Upper(model), [i=1:2], y[i] >= -10)
@constraint(Upper(model), [i=1:2], y[i] <= 20)

# Followed by the objective and constraints of the lower problem:

# Lower objective function
@objective(Lower(model), Min, (-x[1] + y[1] + 40)^2 + (-x[2] + y[2] + 20)^2)

# Lower constraints
@constraint(Lower(model),  [i=1:2],- x[i] + 2y[i] <= -10)
@constraint(Lower(model), [i=1:2], y[i] >= -10)
@constraint(Lower(model), [i=1:2], y[i] <= 20)

# Now we can solve the problem and verify the solution again that reported by the book

optimize!(model)
primal_status(model)
termination_status(model)

# Auto testing
@test objective_value(model) ≈ 0 atol=1e-3
sol = vcat(value.(x), value.(y))
@test sol ≈ [0 ; 0 ; -10; -10] || sol ≈ [0 ; 30; -10; 10] #atol=1e-3

# # Like any other optimization problem, there is a chance in bilevel optimization to find multiple solutions with the same optimal value; based on the inherent stochasticity of the algorithm and random seed, we are expecting two optimal solutions for this problem.


# This file is a part of AstroLib.jl. License is MIT "Expat".
# Copyright (C) 2016 Mosè Giordano.

function _planck_wave{T<:AbstractFloat}(wavelength::T, temperature::T)
    c1  = 3.741771790075259e-16 # = 2*pi*h*c*c
    c2  = 1.43877735382772e-2   # = h*c/k
    return c1/(wavelength^5*(expm1(c2/wavelength/temperature)))
end

"""
    planck_wave(wavelength, temperature) -> black_body_flux

### Purpose ###

Calculate the flux of a black body per unit wavelength.

### Explanation ###

Return the spectral radiance of a black body per unit wavelength using [Planck's
law](https://en.wikipedia.org/wiki/Planck%27s_law)

\$\$ B_\\lambda(\\lambda, T) =\\frac{2hc^2}{\\lambda^5}\\frac{1}{e^{\\frac{hc}{\\lambda k_\\mathrm{B}T}} - 1} \$\$

### Arguments ###

* `wavelength`: wavelength at which the flux is to be calculated, in meters.
* `temperature`: the equilibrium temperature of the black body, in Kelvin.

Both arguments can be either scalar or arrays of the same length.

### Output ###

The spectral radiance of the black body, in units of W/(sr·m³).

### Example ###

Plot the spectrum of a black body in \$[0, 3]\$ µm at \$5000\$ K.  Use
[PyPlot.jl](https://github.com/stevengj/PyPlot.jl) for plotting.

``` julia
using PyPlot
wavelength=linspace(0, 3e-6, 1000);
temperature=ones(wavelength)*5000;
flux=planck_wave(wavelength, temperature);
plot(wavelength, flux)
```

### Notes ###

`planck_freq` calculates the flux of a black body per unit frequency.

Code of this function is based on IDL Astronomy User's Library.
"""
planck_wave(w::Real, t::Real) = _planck_wave(promote(float(w), float(t))...)

@vectorize_2arg Real planck_wave


#!/usr/bin/env julia

"""
# Problem 2: Even Fibonacci numbers

Each new term in the Fibonacci sequence is generated by adding the previous two
terms. By starting with 1 and 2, the first 10 terms will be:

    1, 2, 3, 5, 8, 13, 21, 34, 55, 89, ...

By considering the terms in the Fibonacci sequence whose values do not exceed
`limit`, find the sum of the even-valued terms.
"""
module Problem002

include("Sequences.jl")

"""
    solve(; limit::Integer = 4_000_000)

Returns the solution for [`Problem002`](@ref) with the given parameters.
"""
function solve(; limit::Integer = 4_000_000)
    total = 0
    for (i, n) in enumerate(Sequences.Fibonacci())
        # don't consider terms above limit
        if n > limit
            break
        end

        # sum every third (even) Fibonacci term
        if i % 3 == 2
            total += n
        end
    end
    return total
end

end


# ### Function Contour plot

#md # ![FILE_NAME.png](images/FILE_NAME.png)

## ref https://jp.mathworks.com/matlabcentral/fileexchange/35283-matlab-plot-gallery-function-contour-plot?focused=7851904&tab=example
f(x,y) = sin(3x) * cos(x+y)

x = 0:0.01:5
y = 0:0.01:5
contour(x, y, f, xlabel="x", ylabel="y", fill=true)


using Test, LinearAlgebra
using QBase

@testset "./src/constructors/unitaries.jl" begin

@testset "pauli constants" begin
    @test σI == [1 0;0 1]
    @test σI isa Unitary{Int64}

    @test σx == [0 1; 1 0]
    @test σx isa Unitary{Int64}

    @test σy == [0 -im; im 0]
    @test σy isa Unitary{Complex{Int64}}

    @test σz == [1 0; 0 -1]
    @test σz isa Unitary{Int64}
end

@testset "qubit_rotation()" begin
    ρx = 0.5*[1 1; 1 1]
    ρz = [1 0; 0 0]

    @testset "rotation about x-axis" begin
        for θ in -π:0.1π:π
            Rx = qubit_rotation(θ, axis="x")
            @test isapprox(Rx*ρz*Rx',  [cos(θ/2)^2 im*sin(θ/2)cos(θ/2); -im*sin(θ/2)cos(θ/2) sin(θ/2)^2], atol=1e-7)
        end
    end

    @testset "rotation about y-axis" begin
        for θ in -π:0.1π:π
            Ry = qubit_rotation(θ, axis="y")
            @test isapprox(Ry*ρz*Ry',  [cos(θ/2)^2 sin(θ/2)cos(θ/2); sin(θ/2)cos(θ/2) sin(θ/2)^2], atol=1e-7)
        end
    end

    @testset "rotation about z-axis" begin
        for θ in -π:0.1π:π
            Rz = qubit_rotation(θ, axis="z")
            @test isapprox(Rz*ρz*Rz', ρz)
            @test isapprox(Rz*ρx*Rz', [0.5 0.5*(cos(θ)-im*sin(θ)); 0.5*(cos(θ)+im*sin(θ)) 0.5], atol=1e-7)
        end
    end
end

@testset "random_unitary()" begin
    u2 = random_unitary(2)
    @test u2 isa Unitary{Complex{Float64}}
    @test size(u2) == (2,2)

    u3 = random_unitary(3)
    @test u3 isa Unitary{Complex{Float64}}
    @test size(u3) == (3,3)

    u8 = random_unitary(8)
    @test u8 isa Unitary{Complex{Float64}}
    @test size(u8) == (8,8)

    u3_samples = random_unitary.(fill(3,10))
    @test u3_samples isa Vector{Unitary{Complex{Float64}}}
    @test length(u3_samples) == length(unique(u3_samples))
end

end