module Gallery
import LinearAlgebra
using ..NLEVP

"""
    D,x=cheb(N)

Compute differentiation matrix `D` on Chebyshev grid `x`. See Lloyd N.
Trefethen. Spectral methods in MATLAB. Society for Industrial and Applied
Mathematics, Philadelphia, PA, USA, 2000.
"""
function cheb(N)
    # Compute D = differentiation matrix, x = Chebyshev grid.
    # See Lloyd N. Trefethen. Spectral Methods in MATLAB. Society for Industrial
    # and Applied Mathematics, Philadelphia, PA, USA, 2000.
    if N == 0
        D=0
        x=1
        return D,x
    end

    x = cos.(pi/N*(0:N))
    c = [2.; ones(N-1); 2. ].*((-1.).^(0:N))
    X = repeat(x,1,N+1)
    dX= X-X'
    I=Array{Float64}(LinearAlgebra.I,N+1,N+1)
    D =(c*(1. ./c'))./(dX+I) # off-diagonal entries
    for i = 1:size(D)[1]
        D[i,i]-=sum(D[i,:])
    end
    #D=D -np.diag(np.array(np.sum(D,1))[:,0]) #diagonal entries
    return D,x
end

"""
    L,y=orr_sommerfeld(N=64, Re=5772, w=0.26943)

Return a linear operator family `L` representing the Orr-Sommerfeld equation.
A Chebyshev collocation method is used to discretize the problem. The grid
points are returned as `y`. The number of grid points is `N`.

#Note
The OrrSommerfeld equation reads:
[(d²/dy²-λ²)²-Re((λU-ω)(d²/dy²-λ²)-λU'')] v = 0

where y is the spatial coordinate, λ the wavenumber (set as the eigenvalue by
default), Re the Reynolds number, ω the oscillation frequency, U the mean flow
velocity profile and v the velocity fluctuation amplitude in the y-direction
(the mode shape).

#Remarks on implementation
This is a Julia port of (our python port of) the Orr-Sommerfeld example from the
Matlab toolbox "NLEVP: A Collection of Nonlinear Eigenvalue Problems"
by  T. Betcke, N. J. Higham, V. Mehrmann, C. Schröder, and F. Tisseur.

The original toolbox is available at :
http://www.maths.manchester.ac.uk/our-research/research-groups/numerical-analysis-and-scientific-computing/numerical-analysis/software/nlevp/

See  [1] or [2] for further refrence.

#References
[1] T. Betcke, N. J. Higham, V. Mehrmann, C. Schröder, and F. Tisseur, NLEVP:
A Collection of Nonlinear Eigenvalue Problems, MIMS EPrint 2011.116, December
2011.

[2] T. Betcke, N. J. Higham, V. Mehrmann, C. Schröder, and F. Tisseur, NLEVP:
A Collection of Nonlinear Eigenvalue Problems. Users' Guide, MIMS EPrint
2010.117, December 2011.
"""
function orr_sommerfeld(N=64, Re=5772, ω =0.26943)
    # Define the Orr-Sommerfeld operator for spatial stability analysis.
    # N: number of Cheb points, R: Reynolds number, w: angular frequency
    N=N+1
    # 2nd- and 4th-order differentiation matrices:
    D,y=cheb(N); D2 =D^2;  D2 = D2[2:N,2:N]
    S= LinearAlgebra.diagm(0=>[0; 1.0./(1. .-y[2:N].^2.); 0])
    D4=(LinearAlgebra.diagm(0=>1. .-y.^2.)*D^4-8*LinearAlgebra.diagm(0=>y)*D^3-12*D^2)*S
    D4 = D4[2:N,2:N]

    I=Array{ComplexF64}(LinearAlgebra.I,N-1,N-1)
    # type conversion
    D2=D2.+0.0im
    D4=D4.+0.0im
    U=LinearAlgebra.diagm(0=>-y[2:N].^2.0.+1)
    # Build Orr-Sommerfeld operator
    L= LinearOperatorFamily(["λ","ω","Re","a"],complex([1.,ω,Re,Inf]))
    push!(L,Term(I,(pow_a(4),),((:λ,),),"λ^4","I"))
    push!(L,Term(1.0im*U,(pow_a(3),pow1,),((:λ,),(:Re,),),"iλ^3Re","i*U"))
    push!(L,Term(-2*D2,(pow2,),((:λ,),),"λ^2","-2D2"))
    push!(L,Term(-1.0im*I,(pow2,pow1,pow1,),((:λ,),(:ω,),(:Re,),),"λ^2*ω*Re","-i*I"))
    push!(L,Term(-1.0im*(U*D2+2.0*I),(pow1,pow1,),((:λ,),(:Re,),),"λ*Re","(U*D2+2*I)"))
    push!(L,Term(1.0im*D2,(pow1,pow1),((:ω,),(:Re,),),"ω*Re","i*D2"))
    push!(L,Term(D4,(),(),"","D4"))
    push!(L,Term(-I,(pow1,),((:a,),),"-a","__aux__"))
    return L,y
end

"""
    L,x,y=biharmonic(N=12;scaleX=2,scaleY=1+sqrt(5))

Discretize the biharmonic equation.

# Arguments
-`N::Int`: Number of collocation points
-`scaleX::Float=2` length of the x-axis
-`scaleY::Float=1+sqrt(5)` length of the y-axis

# Returns
-`L::LinearOperatorFamily`: discretization of the biharmonic operator
- `x:Array`: x-axis
- `y:Array`: y-axis

# Notes
The biharmonic equation models the oscillations of a membrane. It is an an eigenvalue problem that reads:
(∇⁴+εcos(2πx)cos(πy))**v**=λ**v** in Ω
with boundary conditions
**v**=∇²**v**=0

The term εcos(2πx)cos(πy) is included to model some inhomogeneous material properties.
The equation is discretized using Chebyshev collocation.

See also: [`cheb`](@ref)
"""

function biharmonic(N=12;scaleX=2,scaleY=1+sqrt(5))
#N,scaleX,scaleY=12,2,1+sqrt(5)
    N=N+1
    D, xx = cheb(N)
    x = xx/scaleX
    y = xx/scaleY

    #The Chebychev matrices are space dependent scale the accordingly
    Dx = D*scaleX
    Dy = D*scaleY

    D2x=Dx*Dx
    D2y=Dy*Dy
    Dx = Dx[2:N,2:N]
    Dy = Dy[2:N,2:N]
    D2x= D2x[2:N,2:N]
    D2y= D2y[2:N,2:N]
    #Apply BC
    I = Array{ComplexF64}(LinearAlgebra.I,N-1,N-1)
    L = kron(I,D2x) +kron(D2y,I)
    X=kron(ones(N-1),x[2:N])
    Y=kron(y[2:N],ones(N-1))
    P=LinearAlgebra.diagm(0=>cos.(π*2*X).*cos.(π*Y))
    D4= L*L
    I = Array{ComplexF64}(LinearAlgebra.I,(N-1)^2,(N-1)^2)
    L= LinearOperatorFamily(["λ","ε","a"],complex([0,0.,Inf]))
    push!(L,Term(D4,(),(),"","D4"))
    push!(L,Term(P,(pow1,),((:ε,),),"ε","P"))
    push!(L,Term(-I,(pow1,),((:λ,),),"-λ","I"))
    push!(L,Term(-I,(pow1,),((:a,),),"-a","__aux__"))
    return L,x,y
end

## 1DRijkeModel
using SparseArrays
"""
    L,grid=rijke_tube(resolution=128; l=1, c_max=2,mid=0)

Discretize a 1-dimensional Rijke tube. The model is based on the thermoacoustic equations.
This eigenvalue problem reads:
∇c²(x)∇p+ω²p-n exp(-iωτ) ∇p(x_ref)=0 for x in ]0,l[
with boundariy conditions
∇p(0)=p(l)=0
"""
function rijke_tube(resolution=127; l=1, c_max=2,mid=0)
  n=1.0
  tau=2
  #l=1
  c_min=1
  #c_max=2
  outlet=resolution
  outlet_c=c_max
  grid=range(0,stop=l,length=resolution)
  e2p=[(i, i+1) for i =1:resolution-1]
  number_of_elements=resolution-1
  if mid==0
    mid=div(resolution,2)+1# this is the element containing the flame
    #it is not the center if resolotion is odd but then the refernce is in the center
  end
  ref=mid-1 #the reference element is the one in the middle
  flame=(mid)
  #compute element volume
  e2v=diff(grid)
  V=e2v[mid]
  e2c=[i < mid ? c_min : c_max for i = 1:resolution]

  #assemble mass matrix
  m_unit=[2 1;1 2]*1/6
  #preallocation
  ii=Array{Int64}(undef,(resolution-1,2^2))
  jj=Array{Int64}(undef,(resolution-1,2^2))
  mm=Array{ComplexF64}(undef,(resolution-1,2^2))
  for (idx,el) in enumerate(e2p)
    mm[idx,:] = m_unit[:]*e2v[idx]
    ii[idx,:] = [el[1] el[2] el[1] el[2]]
    jj[idx,:] = [el[1] el[1] el[2] el[2]]
  end
  # finally assemble global (sparse) mass matrix
  M =sparse(ii[:],jj[:],mm[:])

  # assemble stiffness matrix
  k_unit = -[1. -1.; -1. 1.]
  #preallocation
  ii=Array{Int64}(undef,(resolution-1,2^2))
  jj=Array{Int64}(undef,(resolution-1,2^2))
  kk=Array{ComplexF64}(undef,(resolution-1,2^2))
  for (idx,el) in enumerate(e2p)
    kk[idx,:]=k_unit[:]/e2v[idx]*e2c[idx]^2
    ii[idx,:] = [el[1] el[2] el[1] el[2]]
    jj[idx,:] = [el[1] el[1] el[2] el[2]]
  end
  # finally assemble global (sparse) stiffness matrix
  K = sparse(ii[:],jj[:],kk[:])

  #assemble boundary mass matrix
  bb=[-outlet_c*1im]
  ii=[outlet]
  jj=[outlet]
  B = sparse(ii,jj,bb)
  #assemble flame matrix
  #preallocation
  ii=Array{Int64}(undef,(length(flame),2*2))
  jj=Array{Int64}(undef,(length(flame),2*2))
  qq=Array{ComplexF64}(undef,(length(flame),2*2))

  grad_p_ref=[-1 1]
  grad_p_ref=grad_p_ref/e2v[ref]*1
  #println("##!!!!!Here!!!!!##'")
  #println("flame:",flame)
  #println("ref:",ref)
  for (idx,el) in enumerate(flame)
    jj[idx,:]=[e2p[ref][1] e2p[ref][2] e2p[ref][1] e2p[ref][2]]
    ii[idx,:]=[e2p[el][1] e2p[el][1] e2p[el][2] e2p[el][2]]
    run=1
    for i=1:2
      for j =1:2
        qq[idx,run]= grad_p_ref[1,j]*e2v[el]/2
        run+=1
      end
    end
  end
  #finally assemble flame matrix
  Q= sparse(ii[:],jj[:],-qq[:],resolution,resolution)
  Q=Q/V
  L=LinearOperatorFamily(["ω","n","τ","Y","λ"],complex([0.,n,tau,1E15,Inf]))
  push!(L,Term(M,(pow2,),((:ω,),),"ω^2","M"))
  push!(L,Term(K,(),(),"","K"))
  push!(L,Term(B,(pow1,pow1,),((:ω,),(:Y,),),"ω*Y","C"))
  push!(L,Term(Q,(pow1,exp_delay,),((:n,),(:ω,:τ,)),"n*exp(-i ω τ)","Q"))
  push!(L,Term(-M,(pow1,),((:λ,),),"-λ","__aux__"))
  #push!(L,Term(-SparseArrays.I,(pow1,),((:λ,),),"-λ","__aux__"))
 return L,grid
end

end


using Test
using Plots

include("../../../lib/DifferencesMatrix.jl")
using .DifferencesMatrix: backwardDiff, forwardDiff, centeredDiff

include("../../../lib/SpecialVectors.jl")
using .SpecialVectors: squares

include("../../../lib/KTBC.jl")
using .KTBC: CreateKTBC

# 8
dim = 5
forwardDiff(dim) - backwardDiff(dim)

backwardDiff(dim) * forwardDiff(dim)

backwardDiff(dim) * centeredDiff(dim)

# 10
forwardDiff(dim) * backwardDiff(dim)

function getH(n::Int)
    return 1 / (n + 1)
end

function getAnalogRange()
    return 0:10^(-2):1
end

# 13
function discretSolution(n::Int)
    h = getH(n)
    T = backwardDiff(n) * (-forwardDiff(n))
    u = h^2 * (T \ ones(n))
    return u
end

n = 7
analogSolution(x) = (1 / 2) * (1 - x^2)
steps = 0:1 / (n + 1):1

uₜ = analogSolution.([steps...])[2:(n + 1)]
plot(uₜ)

u₇ = discretSolution(n)
plot!(u₇)

e = uₜ - u₇
plot(e)

# 14
function cosAgainstDiscretCos(n::Int)
    x = getAnalogRange()

    h = getH(n)
    xᵢ = 0:h:1
    ω = 4 * π

    u = cos.(ω * x)
    uᵢ = cos.(ω * h * (1:n + 2))

    lineWidth = 2
    plot(x, u, label="analog cos", lw=lineWidth, title="Cos/DiscretCos Oscilations with N=$n")
    plot!(xᵢ, uᵢ, label="discret cos", lw=lineWidth)
end

cosAgainstDiscretCos(3)
cosAgainstDiscretCos(20)
cosAgainstDiscretCos(150)
cosAgainstDiscretCos(350)

# 16
function analogAndDiscretSolutions(n::Int)
    ω = 4 * π
    h = getH(n)

    rangeₐ = getAnalogRange()

    analogSolution(x) = ((ω)^-2) * cos(ω * x) - ((ω)^-2)
    u = analogSolution.(rangeₐ)

    rangeᵢ = h * (1:n)
    Kₙ, Tₙ, Bₙ, Cₙ = CreateKTBC(n)
    uᵢ = h^2 * (Kₙ \ cos.(ω * rangeᵢ))

    plot(rangeₐ, u, label="analog", title="Analog+Discret solutions with N=$n")
    plot!(rangeᵢ, uᵢ, label="discret")
end

analogAndDiscretSolutions(4)
analogAndDiscretSolutions(8)
analogAndDiscretSolutions(16)
analogAndDiscretSolutions(32)

# 18
function analogAndDiscretSolutions2(n::Int)
    h = getH(n)

    rangeₐ = getAnalogRange()

    analogSolution(x) = (x / 6) * (x^2 - 1)
    u = analogSolution.(rangeₐ)

    rangeᵢ = h * (1:n)
    Kₙ = CreateKTBC(n)[1]
    uᵢ = h^2 * (-Kₙ \ rangeᵢ)

    plot(rangeₐ, u, label="analog", title="Analog+Discret solutions with N=$n")
    plot!(rangeᵢ, uᵢ, label="discret")
end

analogAndDiscretSolutions2(4)
analogAndDiscretSolutions2(8)
analogAndDiscretSolutions2(32)
analogAndDiscretSolutions2(64)

# 19
function analogAndDiscretSolutions3(n::Int)
    h = getH(n)

    rangeₐ = getAnalogRange()

    cₐ = 1 / (ℯ - 1)
    analogSolution(x) = cₐ - cₐ * ℯ^x + x
    u = analogSolution.(rangeₐ)

    f = ones(n)
    ◬₊ = forwardDiff(n)
    ◬₋ = backwardDiff(n)
    ◬₀ = (◬₊ + ◬₋) / (2 * h)
    Kₙ = CreateKTBC(n)[1]
    K = Kₙ / (h^2)

    rangeᵢ = h * (1:n)
    uᵢ = round.((K + ◬₀) \ f, digits=4)
    u₊ = round.((K + ◬₊ / h) \ f, digits=4)

    plot(rangeₐ, u, label="analog", title="Analog+Discret solutions with N=$n")
    plot!(rangeᵢ, uᵢ, label="discret centered")
    plot!(rangeᵢ, u₊, label="discret uncentered")
end

analogAndDiscretSolutions3(4)
analogAndDiscretSolutions3(8)
analogAndDiscretSolutions3(16)
analogAndDiscretSolutions3(32)
analogAndDiscretSolutions3(64)


using Printf

function newton(f, df, p0, n_max, rel_tol; verbose = true)

    converged = false;
    p = p0;
    p_old = p0;

    for i in 1:n_max

        p = p_old - f(p_old)/df(p_old);

        if verbose
            @printf(" %d: p = %.15g, f(p) = %g\n", i, p, f(p));
        end

        if (i>1)
            if abs(p-p_old)/abs(p)< rel_tol
                converged = true;
                break
            end
        end

        p_old = p;

    end

    if !converged
        @printf("ERROR: Did not converge after %d iterations\n", n_max);
    end

    return p

end

function bisection(f, a, b, n_max, rel_tol; verbose = true)

    converged = false;
    p_old = 0;
    p = 0;
    for i in 1:n_max

        p = 0.5 * (a+b)

        if verbose
            @printf(" %d: a = %g, b = %g, p = %g, f(p) = %g\n", i, a, b, p, f(p));
        end

        if ( f(a) * f(p)<=0)
            b = p;
        else
            a = p
        end

        if (i>1)
            if abs(p-p_old)/abs(p)< rel_tol
                converged = true;
                break
            end
        end

        if(abs(f(p))==0)
            converged = true;
            break
        end
        p_old = p;

    end

    if !converged
        @printf("ERROR: Did not converge after %d iterations\n", n_max);
    end

    return p

end

function secant(f, p0, p1, n_max, rel_tol; verbose = true)

    converged = false;

    p = p0;
    for i in 1:n_max

        p = p1 - f(p1) * (p1-p0)/(f(p1)-f(p0));

        if verbose
            @printf(" %d: p = %.12g, f(p) = %g\n", i, p, f(p));
        end

        if (i>1)
            if abs(p-p1)/abs(p1)< rel_tol
                converged = true;
                break
            end
        end
        p0 = p1;
        p1 = p;

    end

    if !converged
        @printf("ERROR: Did not converge after %d iterations\n", n_max);
    end

    return p

end


# # Fidelity in quantum information theory
# This example is inspired from a lecture of John Watrous in the [course on Theory of Quantum Information](https://cs.uwaterloo.ca/~watrous/CS766/LectureNotes/08.pdf).
#
# The Fidelity between two Hermitian semidefinite matrices P and Q is defined as:
#
# $$
# F(P, Q) = \|P^{1/2}Q^{1/2}\|_{\text{tr}} = \max_U \mathrm{tr}(P^{1/2}U Q^{1/2})
# $$
#
# where the trace norm $\|\cdot\|_{\text{tr}}$ is the sum of the singular values, and the maximization goes over the set of all unitary matrices U. This quantity can be expressed as the optimal value of the following complex-valued SDP:
#
# $$
# \begin{array}{ll}
#   \mbox{maximize} &  \frac{1}{2}\text{tr}(Z+Z^\dagger) \\
#   \mbox{subject to} &\\
#   & \left[\begin{array}{cc}P&Z\\{Z}^{\dagger}&Q\end{array}\right] \succeq 0\\
#   & Z \in \mathbf {C}^{n \times n}\\
# \end{array}
# $$
#

using Convex, SCS, LinearAlgebra
if VERSION < v"1.2.0-DEV.0"
     LinearAlgebra.diagm(v::AbstractVector) = diagm(0 => v)
end

n = 20
P = randn(n,n) + im*randn(n,n)
P = P*P'
Q = randn(n,n) + im*randn(n,n)
Q = Q*Q'
Z = ComplexVariable(n,n)
objective = 0.5*real(tr(Z+Z'))
constraint = [P Z;Z' Q] ⪰ 0
problem = maximize(objective,constraint)
solve!(problem, SCSSolver(verbose=0))
computed_fidelity = evaluate(objective)

#-

## Verify that computer fidelity is equal to actual fidelity
P1,P2 = eigen(P)
sqP = P2 * diagm([p1^0.5 for p1 in P1]) * P2'
Q1,Q2 = eigen(Q)
sqQ = Q2 * diagm([q1^0.5 for q1 in Q1]) * Q2'

#-

actual_fidelity = sum(svdvals(sqP * sqQ))

# We can see that the actual fidelity value is very close the computed fidelity value.

docs/examples_literate/optimization_with_complex_variables/Fidelity in Quantum Information Theory.jl


module CalculoNumerico

@doc raw"
	newton_raphson

Esta função implementa uma básica versão do método de Newton-Raphson. Essencialmente, esta implementação depende de um ponto inicial ``x_0``, uma função $f$ e sua derivada $f'$. Estabelecida uma precisão $ε$, o método de Newton-Raphson gera uma sequência iterativa
	$$x_{k+1} = x_{k} -frac{f(x_k)}{f'(x_k)},$$
que, sob certas condições converge para solução do problema.
	Esta função retorna a raiz, o valor da função na raiz e o número de iterações necessárias.

# Exemplo

colocar exemplo
"
function newton_raphson(x0,f,df,ε=10.0^(-4))
	x = x0 - f(x0)/df(x0)
	k = 1
	while abs(x-x0)>=ε && abs(f(x))>ε
		x0 = x
		x = x0 - f(x0)/df(x0)
		k += 1
	end
	return x,f(x),k
end

end # module


# julia1_4.2.jl Vector and Matrix Operations
#
# Most of the math functions work equally well on Vectors
# and matrices.
# The same goes for operations but some need a preceding
# dot (.) to be corresponding element by element ops
# Those include .* .+ .- .% ./ .\ and .^ or any op with otherwise
#  different normal vector/matrix behavior. .+ and .- were enforced
#  recently for clarity of intent.
#  library functions that are operating on element-wise also require a dot
# The following types have a variety of elementary
# operations and functions with optimized methods:
# Type	            Description
# Diagonal	        Diagonal matrix
# Hermitian	        Hermitian matrix - Square, self conjugate
# UpperTriangular	Upper right triangular matrix
# LowerTriangular	Lower left triangular matrix
# Tridiagonal	    Tridiagonal matrix
# SymTridiagonal	Symmetric tridiagonal matrix
# Bidiagonal	    (true)Upper/(false)Lower bidiagonal matrix

# In addition, The LAPACK library is available.
# and a large variety of matrix factorizations

# Examples:
using LinearAlgebra
A = [2.5 2 -4; 3 -1 -6; -10 2.3 4.4]
println(eigvals(A))
println(eigvecs(A))
# 3 ways to get an inverse
Ainv=inv(A)
A^-1
A\I  # I is the identity/uniform-scaling operator
# checking out the inverse
A*Ainv

Am1=A^2 .+ exp.(A) .- 1

B = [2.5 2 -4; 2 -1 -3; -4 -3 4.4]
sB=Symmetric(B)  # less computation
# solve for x in sB*x=v linear system of equations
v=[1.0,2,3]
x=sB\v
println(x)  # Linear Solutions
#
###
#
# more than 12 different factorizations are available
# Single Valued Decomposition SVD
#
(U,S,V)=svd(B)  # such that U*diagm(S)*V'
println(U)      # Matrix
println(V)      # Matrix
println(S)      # Vector diagonal of matrix
println(U*Diagonal(S)*V') # V' transpose of matrix V
#
# LU decomposition L=lower triangular part,
# U=upper triangular part, p= permutation vector
(L,U,p)=lu(B)
println(L)  # lower triangle
println(U)  # upper triangle
println(p)  # permutation vector for rows
println((L*U)[p,:]) # rearrange rows by p

# Make a random Matrix
R=rand(3,3) .+ Diagonal([1.0, 1, 1])  # random matrix R + unit diag. matrix
Ri=inv(R)            # now invert it

# Sparce matrices - type=SparseMatrixCSC Compressed Sparce Cols
#        colVec       rowVec       Data
using SparseArrays
#          row index    col index    values
S=sparse([1, 2, 3, 3],[1, 3, 1, 2],[3, 3, 2, 0])
S=dropzeros(S)
R=S*I
R*rand(3,3)


#script for solving a 2D BVP of the form
# -a0(x)*\Delta u(x,y)+a1(x)*u(x,y)=f(x,y) for (x,y) ∈ Ω:= (0,1)^2
# Here a0(x,y)=1, a1(x,y)=0, f(x,y)=-8*π^2*sin(2*π*x)*sin(2*π*y)
# with Dirichlet boundary conditions u(x,y)=0 for (x,y) ∈ ∂Ω
include("../src/Solver2D.jl")
include("../src/SplinePlotting.jl")

a0(x,y) = 1
a1(x,y) = 0
f(x,y) = 8*π^2*sin(2*π*x)*sin(2*π*y)
exact_sol(x,y) = sin(2*π*x)*sin(2*π*y)
deriv_exact_sol(x,y) = [2*π*cos(2*π*x)*sin(2*π*y), 2*π*sin(2*π*x)*cos(2*π*y)]

#Define the boundary conditions
u_bound(x,y) = 0
bound_left = Boundary2D("Dirichlet", "Left", u_bound)
bound_right = Boundary2D("Dirichlet", "Right", u_bound)
bound_down = Boundary2D("Dirichlet", "Down", u_bound)
bound_up = Boundary2D("Dirichlet", "Up", u_bound)
bound_all = [bound_left, bound_right, bound_down, bound_up]

#Define the domain geometry
cornerLowerLeft = [0., 0.]
lengthx = 1.
lengthy = 1.
uPtRight = 1.
degP = [3, 3]
numSubdiv = 80
nrb = nrbsquare(cornerLowerLeft, lengthx, lengthy, degP, numSubdiv)
#Plots.surface(reuse = false)
#plt = nrbctrlplot(nrb)
#plt = nrbkntplot(nrb)
#display(plt)
IEN, elemVertex = makeIEN(nrb.knots, numSubdiv^2, degP)
IGAmesh = genMesh(nrb)
#plotBasisParam(IGAmesh)

gauss_rule = [genGaussLegendre(degP[1]+1), genGaussLegendre(degP[2]+1)]
stiff = assemble_stiff2D(IGAmesh, a0, gauss_rule)
rhs = assemble_rhs2D(IGAmesh, f, gauss_rule)
bcdof_all, elem_all = classifyBoundary2D(IGAmesh)
lhs, rhs = applyBCnurbs(IGAmesh, bound_all, stiff, rhs, bcdof_all, elem_all, gauss_rule)
@time sol0 = lhs\rhs
plotSol(IGAmesh, sol0, "Poisson2DSquare")
plotSolError(IGAmesh, sol0, exact_sol, "Poisson2DSquareError")
relL2Err, relH1Err = compErrorNorm(IGAmesh, sol0, exact_sol, deriv_exact_sol, a0, gauss_rule)
println("Relative L2-norm error is $relL2Err")
println("Relative energy-norm error is $relH1Err")
print("Done!")


using EngEconomics, Roots, Plots, ColorSchemes
plotly()

uwEngColorScheme = ColorScheme([parse(Colorant, "#D0B4E7"), parse(Colorant, "#57058B"), parse(Colorant, "#8100B4"), parse(Colorant, "#000000")])

# Given
PT = 100
PA = 150
lifeT = 5
lifeA = 5
AT = 50
AA = 62
FT = 20
FA = 30

# Find
# (a.) Construct a break even graph showing the present worth of each
#      alternative as a function of interest rates between 6 percent and 20
#      percent.
i = collect(0.06:0.01:0.2)
NPWT = -PT .+ AT * seriesPresentAmountFactor.(i, lifeT) .+ FT * presentWorthFactor.(i, lifeT)
NPWA = -PA .+ AA * seriesPresentAmountFactor.(i, lifeA) .+ FA * presentWorthFactor.(i, lifeA)

plot(i, NPWT, label="Model T", xlabel="Interest Sensitivity (6%-20%)",
	ylabel="Present Worth (\$1000)",
	title="Sensitivity of Net Present Worth of Models T and A to Interest Rate",
	palette=cgrad(uwEngColorScheme))
plot!(i, NPWA, label="Model A")

# (b.) Which is the preferred choice at 8 percent interest?
i8 = findall(x->x == 0.08, i)
@assert length(i8) == 1
i8 = i8[1]
NPWT_i8 = NPWT[i8]
NPWA_i8 = NPWA[i8]

if NPWT_i8 > NPWA_i8
	println("Model T is preferred because its NPW at i=8% is greater than that of Model A")
else
	println("Model A is preferred because its NPW at i=8% is greater than that of Model T")
end

# (c.) Which is the preferred choice at 16 percent interest?
i16 = findall(x->x == 0.16, i)
@assert length(i16) == 1
i16 = i16[1]
NPWT_i16 = NPWT[i16]
NPWA_i16 = NPWA[i16]

if NPWT_i16 > NPWA_i16
	println("Model T is preferred because its NPW at i=16% is greater than that of Model A")
else
	println("Model A is preferred because its NPW at i=16% is greater than that of Model T")
end

# (d.) What is the break-even interest rate?
NPWT_func(interest) = -PT .+ AT * seriesPresentAmountFactor.(interest, lifeT) .+ FT * presentWorthFactor.(interest, lifeT)
NPWA_func(interest) = -PA .+ AA * seriesPresentAmountFactor.(interest, lifeA) .+ FA * presentWorthFactor.(interest, lifeA)
f(interest) = NPWT_func(interest) - NPWA_func(interest)
iBreakEven = find_zero(f, 0.11)

println("The break even interest rate is roughly ", iBreakEven)


## Exercise 6-5
## The Ackermann function, A(m,n), is defined:

##           ⎧ n + 1  if m = 0
## A(m, n) = ⎨ A(m-1, 1) if m>0 &n = 0
##           ⎩ A(m-1, A(m, n-1)) if m>0 and n>0.

## See https://en.wikipedia.org/wiki/Ackermann_function. Write a function named ack that evaluates the Ackermann function. Use your function to evaluate ack(3, 4), which should be 125. What happens for larger values of m and n?
println("Ans: ")

function ack(m, n)
    if m == 0
        return n + 1
    elseif m > 0 && n == 0
        return ack(m -1, 1)
    elseif m > 0 && n > 0
        return ack(m - 1, ack(m, n -1 ))
    end
end

println(ack(3, 4))

println("End.")


using Test
using Plots

include("../../lib/KTBC.jl")
using .KTBC: CreateKTBC

using LinearAlgebra: I, SymTridiagonal, lu, cholesky, det, eigen, Diagonal, diag, diagm, PosDefException

# 1
K₄ = CreateKTBC(4)[1]

Lₐ = [
  1 0 0 0
  -1 / 2 1 0 0
  0 -2 / 3 1 0
  0 0 -3 / 4 1
]

Uₐ = [
  2 -1 0 0
  0 3 / 2 -1 0
  0 0 4 / 3 -1
  0 0 0 5 / 4
]

L, U = lu(K₄)

@test K₄ == L * U
@test K₄ == L * Diagonal(U) * L'
@test det(K₄) == 5

# 3
K₅ = CreateKTBC(4)[1]

@test round.(det(K₅) * inv(K₅)) == [
 4.0  3.0  2.0  1.0
 3.0  6.0  4.0  2.0
 2.0  4.0  6.0  3.0
 1.0  2.0  3.0  4.0
]

L, U = lu(K₅)
invL = inv(L)

@test K₅ == L * Diagonal(U) * L'

function checkInvL(i::Int, j::Int)
    return round(invL[i, j], digits=2) == round(j / i, digits=2)
end

@test checkInvL(3, 1)
@test checkInvL(3, 2)
@test checkInvL(4, 2)
@test round(invL[4, 2], digits=1) == 1 / 2

# 4
d = (2:5) ./ (1:4)

L = I(4) - diagm(-1 => (1:3) ./ (2:4))

L * Diagonal(d) * L'

# 9
K₃, T₃, B₃, C₃ = CreateKTBC(3)

A = cholesky(convert(Array{Float64}, K₃))
@test round.(A.L * A.U) == K₃

A = cholesky(convert(Array{Float64}, T₃))
@test round.(A.L * A.U) == T₃

@test_throws PosDefException cholesky(convert(Array{Float64}, B₃))

Bₑ = B₃ + eps() * I(3)
A = cholesky(convert(Array{Float64}, Bₑ))
@test round.(A.L * A.U) == round.(Bₑ)

# 10
eye₄ = ones(4, 4)
eigen(eye₄)

@test det(eye₄) == 0.0

L = [
  1
  1
  1
  1
]

@test L * L' == eye₄

# 11
K = ones(4, 4) + I(4) / 100
L, U = lu(K)
eigen(K)
inv(K)

# 12
# https://rosettacode.org/wiki/Pascal_matrix_generation#Julia
function pascal(n::Int)
    return [binomial(j + i, i) for i in 0:n, j in 0:n]
end

# https://en.wikipedia.org/wiki/Pascal_matrix
function pascalL(n::Int)
    return exp(diagm(-1 => 1:(n - 1)))
end

K = pascal(4)
L, U = lu(K)

@test det(K) == abs(reduce(*, diag(U)))

L = pascalL(5)

@test K == round.(L * L')

# 13
Fib = [
  1 1
  1 0
]

v = [
  1
  0
]

@test Fib^6 * v == [
  13
  8
]

# 14
L = [
  1 0 0
  3 1 0
  0 2 1
]

U = [
  2 8 0
  0 3 5
  0 0 7
]

f = [
  0
  3
  6
]

c = L \ f
x = U \ c

@test x == round.(L * U \ f)


## In this notebook, we'll look at least absolute deviation on biometric data

##
using Plots
pyplot()
theme(:dark)
##
# https://vincentarelbundock.github.io/Rdatasets/csv/carData/Davis.csv
using CSV
df = CSV.read(download("https://vincentarelbundock.github.io/Rdatasets/csv/carData/Davis.csv"))
X = Float64.([df[:weight] df[:height]])
# Filter the data
goodpts = X[:,1] .<= 115
Xf = X[goodpts,:]
##
scatter(Xf[:,1],Xf[:,2],label="",xlabel="weight (kg)", ylabel="height (cm)")
##
x = Xf[:,1]
A = [x ones(length(x))]
b = Xf[:,2]
ab_ls = A\b # solve the least squares problem

## Solve the Least Absolute Deviation version
using Convex, SCS
xvar = Variable(2)
problem = minimize(sum(abs(A * xvar - b)))
solve!(problem, SCSSolver(max_iters=20000))
ab_lad = xvar.value
##
using Printf
scatter(Xf[:,1],Xf[:,2],color=2,size=(300,300), label="")
plot!(x, x*ab_ls[1] .+ ab_ls[2], label="",color=5)
plot!(x, x*ab_lad[1] .+ ab_lad[2], label="",color=6)
title!(@sprintf("LS: height ≈ %.2f weight + %.2f\nLAD:  height ≈ %.2f weight + %.2f    ",
ab_ls[1],ab_ls[2],ab_lad[1],ab_lad[2]), titlefontsize=12)
xlabel!("Weight (kg)")
ylabel!("Height (cm)")
savefig("weight-height-fit-lad.pdf")
## Except, we actually filtered the data.
# to remove an outlier
scatter(X[:,1],X[:,2],color=2,size=(300,300), label="",xlabel="weight (kg)", ylabel="height (cm)")
savefig("weight-height-outliers.pdf")
##
x = X[:,1]
A = [x ones(length(x))]
b = X[:,2]
ab_ls = A\b # solve the least squares problem
xvar = Variable(2)
problem = minimize(sum(abs(A * xvar - b)))
solve!(problem, SCSSolver(max_iters=20000))
ab_lad = xvar.value
scatter(X[:,1],X[:,2],color=2,size=(300,300), label="")
plot!(x, x*ab_ls[1] .+ ab_ls[2], label="",color=5)
plot!(x, x*ab_lad[1] .+ ab_lad[2], label="",color=6)
title!(@sprintf("LS: height ≈ %.2f weight + %.2f\nLAD:  height ≈ %.2f weight + %.2f    ",
ab_ls[1],ab_ls[2],ab_lad[1],ab_lad[2]), titlefontsize=12)
xlabel!("Weight (kg)")
ylabel!("Height (cm)")
savefig("weight-height-outliers-fit-lad.pdf")


#############################
##  These numerical experiments, included for the sake of completion,
##  showcase some numerical solutions to LINEAR Volterra integral equations
#############################

using ApproxFun, MultivariateOrthogonalPolynomials, BandedMatrices, BlockBandedMatrices, SpecialFunctions, Plots
using SparseVolterraExamples

#####################################################
## Solving Equation 18 in https://doi.org/10.1137/19M1267441
## This is a linear Volterra integral equation of FIRST kind.
#####################################################
####
## Set up g(x) and kernel K(x,y)
g1(x)= exp(-x)/4+1/4*exp(x)*(-1+2*x)
K(x,y) = exp(y-x)
####
## For linear problems the solver is wrapped into convenient functions. N is polynomial order of approximation.
N = 20
u = triVolterraEQ1FullKernelSolver(g1,K,N,true)
####
## Plot the computed and analytic solution
plot(u,grid=false,ylabel="u(x)",xlabel="x",label="sparse method")
plot!(x->x*exp(x),0,1,grid=false,ylabel="u(x)",xlabel="x",label="analytic")
####
## Plot error
plot(triVolterraEQ1FullKernelSolver(g1,K,N,true)-Fun(x->x*exp(x),Jacobi(0,1,0..1)),ylabel="error",xlabel="x",grid=false,label=false)

#####################################################
## The following is a simple linear second kind Volterra integral equation, not appearing in either paper.
## It uses the monomial coefficient vector input kernel implementation and limits of integration to 1-x instead and has analytically known solution sin(-2*pi*x).
#####################################################
####
## Set up g(x) and kernel K(x,y)
g2 = Fun(x->(x-1)/(2*pi)*cos(-2*pi*x)+(4*pi^2+1)/(4*pi^2)*sin(2*pi*(1-x)), Jacobi(0,1,0..1))
K2 = [0.,1.,0.,0.] # this is equivalent to K(x,y)=y
####
## Compute and plot solutions
N = 20
u = triVolterraEQ2Solver(g2,K2,N,false) # note the false indicating that we are not flipping the integration bounds, i.e. we are integrating from 0 to 1-x
plot(u,grid=false,ylabel="u(x)",xlabel="x",label="sparse method")
plot!(x->sin(-2*pi*x),0,1,grid=false,ylabel="u(x)",xlabel="x",label="analytic")
####
## Plot errors
plot(triVolterraEQ2Solver(g2,K2,N,false)-Fun(x->sin(-2*pi*x),Jacobi(0,1,0..1)),grid=false,ylabel="error",xlabel="x",label=false)
####
## The above is equivalent to the following much simpler input, which uses the full kernel input as a function
u2 = triVolterraEQ2FullKernelSolver(g2,(x,y)->y,N,false)
####
## Verify the two input methods give the same answer
plot(x->abs(u2(x)-u(x)),0,1,grid=false,ylabel="difference",xlabel="x",label=false)

examples/Example 0 - Some linear Volterra integral equation examples.jl


using SparseArrays, DiffEqOperators, LinearAlgebra, Random,
      Test, BandedMatrices, FillArrays

import DiffEqOperators: BoundaryPaddedVector

function second_derivative_stencil(N)
  A = zeros(N,N+2)
  for i in 1:N, j in 1:N+2
      (j-i==0 || j-i==2) && (A[i,j]=1)
      j-i==1 && (A[i,j]=-2)
  end
  A
end

# Analytic solutions to higher order operators.
# Do not modify unless you are completely certain of the changes.
function fourth_deriv_approx_stencil(N)
    A = zeros(N,N+2)
    A[1,1:8] = [3.5 -56/3 42.5 -54.0 251/6 -20.0 5.5 -2/3]
    A[2,1:8] = [2/3 -11/6 0.0 31/6 -22/3 4.5 -4/3 1/6]

    A[N-1,N-5:end] = reverse([2/3 -11/6 0.0 31/6 -22/3 4.5 -4/3 1/6], dims=2)
    A[N,N-5:end] = reverse([3.5 -56/3 42.5 -54.0 251/6 -20.0 5.5 -2/3], dims=2)

    for i in 3:N-2
        A[i,i-2:i+4] = [-1/6 2.0 -13/2 28/3 -13/2 2.0 -1/6]
    end
    return A
end

function second_deriv_fourth_approx_stencil(N)
    A = zeros(N,N+2)
    A[1,1:6] = [5/6 -1.25 -1/3 7/6 -0.5 5/60]
    A[N,N-3:end] = reverse([5/6 -1.25 -1/3 7/6 -0.5 5/60], dims=2)
    for i in 2:N-1
        A[i,i-1:i+3] = [-1/12 4/3 -5/2 4/3 -1/12]
    end
    return A
end

# The following test-sets test for the correctness of the convolutions against
# AbstractVector and BoundaryPaddedArray in one dimension.

@testset "L.boundary_point_count is zero" begin
    N = 20
    L = CenteredDifference(2, 2, 1.0, N)
    cl = rand()
    cr = rand()
    u = rand(N)
    Qu = BoundaryPaddedVector(cl, cr, u)
    arrayQu = [cl;u;cr]

    # Check that BoundaryPaddedVector is constructed correctly
    @test Qu == arrayQu

    # Test for correctness of DerivativeOperator*AbstractVector
    @test second_derivative_stencil(N)*arrayQu ≈ L*arrayQu

    # Test for correctness of DerivativeOperator*BoundaryPaddedVector
    @test second_derivative_stencil(N)*arrayQu ≈ L*Qu
end

@testset "Fourth Derivative and Approximation Order Test" begin
    N = 20
    L = CenteredDifference(4, 4, 1.0, N)
    cl = rand()
    cr = rand()
    u = rand(N)
    Qu = BoundaryPaddedVector(cl, cr, u)
    arrayQu = [cl;u;cr]

    # Check that BoundaryPaddedVector is constructed correctly
    @test Qu == arrayQu

    # Test for correctness of DerivativeOperator*AbstractVector
    @test fourth_deriv_approx_stencil(N)*arrayQu ≈ L*arrayQu

    # Test for correctness of DerivativeOperator*BoundaryPaddedVector
    @test fourth_deriv_approx_stencil(N)*arrayQu ≈ L*Qu
end


begin
  import Pkg; Pkg.activate(".")
  Pkg.add("DifferentialEquations")
  Pkg.add("Optim")
end

using DifferentialEquations, Plots

function f(du,u,p,t)
  du[1] = u[2]
  du[2] = -p
end

u₀ = [50.0,0.0]
tspan = (0.0,15.0)
p = 9.8

prob = ODEProblem(f,u₀,tspan, p)
sol = solve(prob,Tsit5())
plot(sol, label = ["Displacement" "Velocity"],
  xlabel="Time (s)",
  ylabel="Displacement (m) | Velocity (m/s)")

# But where is the ground?
function condition(u,t,integrator) # Event when event_f(u,t) == 0
  u[1]
end

function affect!(integrator)
  integrator.u[2] = -integrator.u[2]
end

cb = ContinuousCallback(condition,affect!, save_positions=(false, false))

sol = solve(prob,Tsit5(), callback=cb)
plot(sol, label = ["Displacement" "Velocity"],
  xlabel="Time (s)",
  ylabel="Displacement (m) | Velocity (m/s)")

### Air resistance
function f₂(du,u,p,t)
  g, Cd = p
  du[1] = u[2]
  du[2] = -(g+u[2]^2*Cd*sign(u[2]))
end

p = (9.81, 15e-2)

prob = ODEProblem(f₂, u₀,tspan, p)
sol = solve(prob, Tsit5(), callback=cb)

plot(sol, label = ["Displacement" "Velocity"],
  xlabel="Time (s)",
  ylabel="Displacement (m) | Velocity (m/s)")

### Optim.jl
using Optim

function compute_ODE(p; times=1:0.5:15)
  prob = ODEProblem(f₂, u₀,tspan, p)
  sol = solve(prob, Tsit5(), saveat=times, callback = cb)
  return sol.t , hcat(sol.u...)[1,:]
end

time, exp_disp = compute_ODE((9.81, 5e-2))
exp_disp .+= 0.75randn(length(exp_disp)) # add noise

scatter(time, exp_disp, label="Experiment", xlabel="Time(s)", ylabel="Displacement (m)")

function objective(Cd)
  _, sim_disp = compute_ODE((9.81, Cd[1]))
  return sum((sim_disp.- exp_disp).^2)
end

res = optimize(objective, [5e-3], Newton(), Optim.Options(store_trace=true, extended_trace=true))
Cd = res.minimizer[1]

plot!(compute_ODE((9.81, Cd); times=0:0.1:15), label="Simulated")

### Visual
for iteration in res.trace[1:10]
  Cd = iteration.metadata["x"][1]
  pl = scatter(time, exp_disp, label="Experiment", xlabel="Time(s)", ylabel="Displacement (m)")
  plot!(pl, compute_ODE((9.81, Cd); times=0:0.1:15), label="Simulated (Cd: $Cd")
  display(pl)
  sleep(1.0)
end


x = rand(1:100, 5)
println("Sum of ", x, " = ", sum(x))

A = rand(-100:100, 3, 5)

sum(A, dims=1)

sum(A, dims=2)

sum(A, dims=1)[:]

x = rand(1:100, 5)
display(x)
prod(x)                         # Product of all elements

display(A)
display(maximum(A, dims=1))     # Largest element in each column
display(minimum(A, dims=1))     # Smallest element in each column

x = 1:5
display(cumsum(x))                   # Cumulative sum, that is, entry n is the sum of x_1,...,x_n
display(cumprod(x))                  # Cumulative product

A = reshape(1:6, 2, 3)

cumsum(A, dims=1)                    # Cumulative sum along dimension 1 (that is, column-wise)

cumprod(A, dims=2)                   # Cumulative product along dimension 2

function taylor_cos(x,n)
    factors = [ -x^2 / ((2k-1) * 2k) for k = 1:n ]
    y = 1 + sum(cumprod(factors))
end

println(taylor_cos(10, 50)) # Taylor approximation
println(cos(10))            # true value

xx = -10:0.01:10
yy = cos.(xx)
yy_taylor = [ taylor_cos(x,n) for x in xx, n in 0:5 ]

using PyPlot
plot(xx, yy, linewidth=2, color="k")
plot(xx, yy_taylor)
grid(true)
axis([-10,10,-1.2,1.2]);

x = 1:2:1000                 # Odd numbers
println(503 in x)            # True, 503 is in the list
println(1000 ∈ x)            # False, 1000 is not in the list

println(all(x .< 500))
println(all(x .> 0))

println(any(x .== 503))
println(any(x .== 1000))

idx = findfirst(x .== 503)    # Index in x with the value 503

ind = findall(@. x % 97 == 0)
println("Indices in x with numbers that are multiples of 97: ", ind)

count(@. x % 97 == 0)


# Load X and y variable
using JLD
data = load("logisticData.jld")
(X,y) = (data["X"],data["y"])

# Add bias variable, initialize w, set regularization and optimization parameters
(n,d) = size(X)
X = [ones(n,1) X]
d += 1
w = zeros(d,1)
lambda = 1
maxPasses = 500
progTol = 1e-4
verbose = true

## Run and time coordinate descent to minimize L2-regularization logistic loss

# Start timer
tic()

# Compute Lipschitz constant of 'f'
(eigenvalues,~) = eig(X'*X)
L = .25*maximum(eigenvalues) + lambda;

# Start running coordinate descent
w_old = copy(w);
for k in 1:maxPasses*d

    # Choose variable to update 'j'
    j = rand(1:d)

    # Compute partial derivative 'g_j'
    yXw = y.*(X*w);
    sigmoid = 1./(1+exp.(-yXw));
    g = -X'*(y.*(1-sigmoid)) + lambda*w;
    g_j = g[j];

    # Update variable
    w[j] -= (1/L)*g_j;

    # Check for lack of progress after each "pass"
    if mod(k,d) == 0
        yXw = y.*(X*w)
        f = sum(log.(1 + exp.(-y.*(X*w)))) + (lambda/2)norm(w)^2
        delta = norm(w-w_old,Inf);
        if verbose
            @printf("Passes = %d, function = %.4e, change = %.4f\n",k/d,f,delta);
        end
        if delta < progTol
            @printf("Parameters changed by less than progTol on pass\n");
            break;
        end
        w_old = copy(w);
    end

end

# End timer
toc()


A = rand(-4:4,5,3)
B = rand(-4:4,3,4)

C = A*B

C[2,3]

m = 5; n = 3; p = 4;
C = zeros(m,p);
for i = 1:m
    for j = 1:p
        for k = 1:n
            C[i,j] += A[i,k]*B[k,j]
        end
    end
end

C

m = 5; n = 3; p = 4;
C = zeros(m,p);
for i = 1:m
    for j = 1:p
        C[i,j] = sum( A[i,k]*B[k,j] for k = 1:n );
    end
end
C

m = 5; n = 3; p = 4;
C = [ sum( A[i,k]*B[k,j] for k = 1:n ) for i=1:m, j=1:p ]

# C is a matrix of inner products
i=2; j=3;
using LinearAlgebra
dot(A[i,:],B[:,j])
C = [ dot(A[i,:],B[:,j]) for i=1:m, j=1:p ]

# C columnwise is a linear combination of columns
C[:,j]
sum( B[k,j]*A[:,k] for k=1:n  )
hcat( [ sum( B[k,j]*A[:,k] for k=1:n  ) for j=1:p ]... )
q = [ sum( B[k,j]*A[:,k] for k=1:n  ) for j=1:p ];
hcat(q[1],q[2],q[3],q[4])

length(C)

length(q)

# C is a sum of outer products
sum( A[:,k]*B[k,:]' for k=1:n)

B[1,:]'

(B[1,:]')'


struct LineSearchOptions
    sufficient_decrease::Float64
    curvature::Float64
    max_iter::Int
end

function LineSearchOptions(; sufficient_decrease = 1e-4, curvature = 0.1, max_iter = 100)
    LineSearchOptions(sufficient_decrease, curvature, max_iter)
end

function line_search(
    fcn,
    x0,
    step_direction,
    f0 = fcn(x0),
    search_options::LineSearchOptions = LineSearchOptions())
# suff_decrease = 1e-4, curvature = 0.1, max_iter = 100)

    # rename for ease of refactoring
    suff_decrease = search_options.sufficient_decrease
    curvature     = search_options.curvature
    max_iter      = search_options.max_iter

    # strong wolfe condition
    step_size_prev = zero(eltype(x0))
    step_size      = one(eltype(x0))
    step_size_lo   = step_size_prev
    step_size_hi   = step_size_prev
    f_prev         = NaN
    dphi0          = numder.directional_diff_fw(fcn, x0, step_direction)

    # bracket phase
    for iter = 1:max_iter

        f1 = fcn(x0 + step_direction * step_size)
        if f1 > f0 + suff_decrease * step_size * dphi0 || (iter > 1 && f1 >= f_prev)
            step_size_lo, step_size_hi = step_size_prev, step_size
            break
        end

        dphi = numder.directional_diff_fw(fcn, x0 + step_direction * step_size, step_direction)
        if abs(dphi) <= abs(dphi0) * curvature
            return step_size
        elseif dphi >= 0
            step_size_lo, step_size_hi = step_size, step_size_prev
            break
        end

        f_prev = f1
        step_size_prev, step_size = step_size, step_size * 2
    end

    # zoom phase
    f_lo = fcn(x0 + step_size_lo * step_direction)
    for iter = 1:max_iter
        step_size = 0.5 * (step_size_lo + step_size_hi)
        f1 = fcn(x0 + step_size * step_direction)
        if f1 > f0 + suff_decrease * step_size * dphi0 || f1 >= f_lo
            step_size_hi = step_size
        else
            dphi = numder.directional_diff_fw(fcn, x0 + step_direction * step_size, step_direction)
            if abs(dphi) <= abs(dphi0) * curvature
                return step_size
            elseif dphi * (step_size_hi - step_size_lo) >= 0
                step_size_hi = step_size_lo
            end
            step_size_lo = step_size
        end

    end

    return step_size
end

#=
function armijo_backtracking(fcn, x0, direction, f0 = fcn(x0), alpha_0 = 1, step_factor = 0.5, max_iteration = 20, beta = 1e-4)
    # sufficient decrease condition shall be satisfied
    phi_0 = f0
    dphi_dalpha_0 = NumDiff.directional_diff_fw(fcn, x0, direction)

    alpha = alpha_0
    for iter = 1:max_iteration
        phi_alpha = fcn( x0 + direction .* alpha)
        if phi_alpha <= phi_0 + beta * alpha * phi_alpha
            break
        end
        alpha = alpha * step_factor
    end
end
=#


## Exercise 7-2
## Copy the loop from Square Roots and encapsulate it in a function called mysqrt that takes a as a parameter, chooses a reasonable value of x, and returns an estimate of the square root of a.

function mysqrt(a, x)
    a = float(a)
    while true
        y = (x + a/x) / 2
        if y == x
            break
        end
        x = y
    end
    return x
end

## To test it, write a function named testsquareroot that prints a table like this:
# see ex 7-2 in book
println("Ans: ")

function rightjustify(string::String, chr)
    if chr - length(string) > 0
        return ' ' ^ (chr - length(string)) * string
    end
    return string
end

function testsquareroot()
    println("a   mysqrt             sqrt               diff")
    println("-   ------             ----               ----")
    for i in 1:10
        i = float(i)
        root = sqrt(i)
        root_my = mysqrt(i, i/2)
        println(
            string(i) * " " *
            rightjustify(string(root_my), 18) *
            " " *
            rightjustify(string(root), 18) * " " * string(abs(root_my - root)))
    end
end

testsquareroot()

println("End.")


# hann.jl
# Hann window function

"""
    hann(x::AbstractFloat)
Hann window function (often called "Hanning" window function).
This version is `0.5 * (1 + cospi(2x))` and used in `[-0.5,+0.5]`.
https://en.wikipedia.org/wiki/Hann_function
"""
hann(x::AbstractFloat) = 0.5 * (1 + cospi(2x))

"""
    hann(n::Int)
Return vector of `n ≥ 2` samples of Hann window function
equally spaced over `[-0.5,+0.5]`.
The first and last samples are zero for `n ≥ 2`.

Caution.
Matlab has both `hann` and `hanning` functions for which
`[0; hanning(n); 0] == hann(n+2)`.

`DSP.hanning(n)` == Matlab.hann(n)`
whereas
`Sound.hann(n) == Matlab.hanning(n)`.
"""
hann(n::Int) = hann.(LinRange(-0.5, 0.5, n+2))[2:end-1]


using Random, Distributions, KernelDensity, Plots, LaTeXStrings; pyplot()
Random.seed!(1)

function tStat(mu0,mu,sig,n)
    sample = rand(Normal(mu,sig),n)
    xBar   = mean(sample)
    s      = std(sample)
    (xBar-mu0)/(s/sqrt(n))
end

mu0, mu1A, mu1B = 20, 22, 24
sig, n = 7, 5
N = 10^6
alpha = 0.05

dataH0  = [tStat(mu0,mu0,sig,n) for _ in 1:N]
dataH1A = [tStat(mu0,mu1A,sig,n) for _ in 1:N]
dataH1B = [tStat(mu0,mu1B,sig,n) for _ in 1:N]
dataH1C = [tStat(mu0,mu1B,sig,2*n) for _ in 1:N]
dataH1D = [tStat(mu0,mu1B,sig/2,2*n) for _ in 1:N]

tCrit = quantile(TDist(n-1),1-alpha)
estPwr(sample) = sum(sample .> tCrit)/N

println("Rejection boundary: ", tCrit)
println("Power under H0:  ", estPwr(dataH0))
println("Power under H1A: ", estPwr(dataH1A))
println("Power under H1B (mu's farther apart): ", estPwr(dataH1B))
println("Power under H1C (double sample size): ", estPwr(dataH1C))
println("Power under H1D (like H1C but std/2): ", estPwr(dataH1D))

kH0  = kde(dataH0)
kH1A = kde(dataH1A)
kH1D = kde(dataH1D)
xGrid = -10:0.1:15

plot(xGrid,pdf(kH0,xGrid),
	c=:blue, label="Distribution under H0")
plot!(xGrid,pdf(kH1A,xGrid),
	c=:red, label="Distribution under H1A")
plot!(xGrid,pdf(kH1D,xGrid),
	c=:green, label="Distribution under H1D")
plot!([tCrit,tCrit],[0,0.4],
	c=:black, ls=:dash, label="Critical value boundary",
	xlims=(-5,10), ylims=(0,0.4), xlabel=L"\Delta = \mu - \mu_0")


using SpecialFunctions, Distributions, Roots, Plots, LaTeXStrings; pyplot()

eq(alpha, xb, xbl) = log(alpha) - digamma(alpha) - log(xb) + xbl

actualAlpha, actualLambda = 2, 3
gammaDist = Gamma(actualAlpha,1/actualLambda)

function mle(sample)
    alpha  = find_zero( (a)->eq(a,mean(sample),mean(log.(sample))), 1)
    lambda = alpha/mean(sample)
    return [alpha,lambda]
end

N = 10^4

mles10   = [mle(rand(gammaDist,10)) for _ in 1:N]
mles100  = [mle(rand(gammaDist,100)) for _ in 1:N]
mles1000 = [mle(rand(gammaDist,1000)) for _ in 1:N]

scatter(first.(mles10), last.(mles10),
	c=:blue, ms=1, msw=0, label="n = 10")
scatter!(first.(mles100), last.(mles100),
	c=:red, ms=1, msw=0, label="n = 100")
scatter!(first.(mles1000), last.(mles1000),
	c=:green, ms=1, msw=0, label="n = 1000",
	xlims=(0,6), ylims=(0,8), xlabel=L"\alpha", ylabel=L"\lambda")


export dde_stability

"""
    dde_stability(A, B, τ; N=10)

Calculate (a finite number of) the characteristic roots of the linear delay differential
equation

```math
x'(t) = Ax(t) + Bx(t-τ)
```

where `A` and `B` can be (square) matrices. `N` controls the order of the Chebyshev
polynomial approximation to the infinitesimal generator; higher values of `N` result in a
more accurate approximation. A property of the algorithm used is that characteristic roots
closest to the origin are approximated best.

## Reference

Pseudospectral Differencing Methods for Characteristic Roots of Delay Differential
Equations; D. Breda, S. Maset, and R. Vermiglio, SIAM Journal on Scientific Computing 2005
27:2, 482-495
"""
function dde_stability(A, B, τ; N=10)
    @assert size(A) == size(B)
    @assert size(A, 1) == size(A, 2)
    T = eltype(A) == Int ? Float64 : eltype(A)
    n = size(A, 1)
    (x, D) = cheb_diff(T, N)
    M = kron(-D, Matrix(I / convert(T, τ), n, n))
    M[1:n, 1:n] .= A
    M[1:n, (n + 1):(end - n)] .= 0
    M[1:n, (end - n + 1):end] .= B
    return eigen(M)
end

# Tests
# 1. x'(t) = 2x(t) - exp(1)*x(t - 1) has a double root at λ = 1
# 2. x'(t) = -x(t - τ) goes unstable at τ=π/2
# 3. x'(t) = α*A*x(t) + (1-α)*A*x(t-τ)
#    A = [
#          50  284   41   23   50   32
#        -280  -46  -19  -37  -10  -28
#          35   -1   26  143   35   17
#          5   -31 -139  -22    5  -13
#          20  -16   11   -7 -115  137
#         -10  -46  -19  -37 -145 -163
#    ]
#    with α=0.9 is stable for 3π/(270*(2*0.9 - 1)) ≤ τ ≤ 4π/270
#    that is 0.04363323129985824 ≤ τ ≤ 0.046542113386515455


using Test, SpikeSynchrony

####### ANALYTICAL TEST ########
y1 = [1]
y2 = [2]
ta = [0, 0, 1, 1, 2, 2, 3, 3]
# I calculated S by hand for these trains:
Sa = [0, 0, 5//9, 3//8, 3//8, 5//9, 0, 0]

t, S = SPIKE_distance_profile(y1, y2; tf = 3)
@test t == ta
@test S ≈ Sa    atol = 1e-15

D_S_a = sum( 1//2 * (Sa[2:end] .+ Sa[1:end-1]) .* (ta[2:end] .- ta[1:end-1]))//(ta[end] - ta[1])
D_S = SpikeSynchrony.trapezoid_integral(t, S)/(t[end] - t[1])
@test D_S ≈ D_S_a    atol = 1e-15

####### Second Analytic test ########
y1 = [2.0, 5.0, 8.0]
y2 = [1, 5, 9]
t, S  = SPIKE_distance_profile(y1, y2; t0 = 0, tf = 10)

analytic_t = [0, 1, 2, 5, 8, 9, 10]
analytic_S = [0, 0, 0.555555555555555580,  0.222222222222222210,
11/36, 25/98, 0, 0, 25/98, 11/36, 0.222222222222222210,  0.555555555555555580, 0, 0]

@test S ≈ analytic_S  atol = 1e-15

####### TEST WITH ARBITRARY REAL DATA ########
t0 = 0; tf = 500
y1 = cumsum(rand(10:10:tf, 100))
y2 = cumsum(rand(10:10:tf, 100))
t, S  = SPIKE_distance_profile(y1, y2)
t, S2 = SPIKE_distance_profile(y2, y1)
t, S0 = SPIKE_distance_profile(y1, y1)
# tr, Sr = SPIKE_distance_profile(abs.(reverse(y1) .- y1[end]), abs.(reverse(y1) .- y1[end]),

# Fundamental tests from the definition of S and D_S
@test issorted(t)
@test (0 .≤ S) == trues(length(S))
@test S == S2
@test S0 == zeros(length(S0))

D_S = SpikeSynchrony.trapezoid_integral(t, S)/(t[end]-t[1]) # == SPIKE_distance(y1, y2)
@test 0 ≤ D_S ≤ 1


using LinearAlgebra
using Statistics: mean, median

#TODO
# R2-Score, Variance-Score, Deviance related functions

export max_error, mean_absolute_error, mean_squared_error, mean_squared_log_error, median_absolute_error, mean_absolute_percentage_error

"""
```max_error(y_true, y_pred)```
Return the maximum residual error.

## Arguments
- `y_true` : Ground truths
- `y_pred` : Predictions

"""
max_error(y_true, y_pred) = maximum(abs.(convert_1d(y_true) .- convert_1d(y_pred)))

"""
```mean_absolute_error(y_true, y_pred; keywords)```
Return the mean absolute error.

## Arguments
- `y_true` : Ground truths
- `y_pred` : Predictions

## Keywords
- `weights` : Sample weights

"""
function mean_absolute_error(y_true, y_pred; weights = nothing)
    _validate_distance_input(y_true, y_pred, weights)
    result = abs.(y_true .- y_pred)
    result = weights == nothing ? mean(result) : mean(result .* weights)
    return result
end

"""
```mean_squared_error(y_true, y_pred; keywords)```
Return the mean squared error.

## Arguments
- `y_true` : Ground truths
- `y_pred` : Predictions

## Keywords
- `weights` : Sample weights
- `squared = false` : Denotes whether or not to execute the last squaring operation
"""
function mean_squared_error(y_true, y_pred; weights = nothing, squared = false)
    _validate_distance_input(y_true, y_pred, weights)
    result = (y_true .- y_pred) .^ 2
    result = !squared ? sqrt.(result) : result
    return weights == nothing ? mean(result) : mean(result .* weights)
end

"""
```mean_squared_log_error(y_true, y_pred; keywords)```
Return the mean squared log error.

## Arguments
- `y_true` : Ground truths
- `y_pred` : Predictions

## Keywords
- `weights` : Sample weights

"""
mean_squared_log_error(y_true, y_pred; weights = nothing) = mean_squared_error(log.(1 .+ y_true), log.(1 .+ y_pred); weights = weights)

"""
```median_absolute_error(y_true, y_pred; keywords)```
Return the median absolute error.

## Arguments
- `y_true` : Ground truths
- `y_pred` : Predictions

## Keywords
- `weights` : Sample weights

"""
function median_absolute_error(y_true, y_pred; weights = nothing)
    _validate_distance_input(y_true, y_pred, weights)
    result = median(abs.(y_pred .- y_true) ; dims = 1)
    return weights == nothing ? mean(result) : mean(result .* weights)
end

"""
```mean_absolute_percentage_error(y_true, y_pred; keywords)```
Return the median absolute percentage error.

## Arguments
- `y_true` : Ground truths
- `y_pred` : Predictions

## Keywords
- `weights` : Sample weights

"""
function mean_absolute_percentage_error(y_true, y_pred; weights = nothing)
    _validate_distance_input(y_true, y_pred, weights)
    result = abs.(y_true .- y_pred) ./ max.(abs.(y_true),eps())
    return weights == nothing ? mean(result) : mean(result .* weights)
end


# Inverse and soln of matrix
# Gaussian
# generally JL handles eqns and invs

# we use Finverse to denote function to find inverse
# 🙈

function Finverse_Gaussian(x)
    n = Int(sqrt(length(x)))
    x = reshape(x,n,n)
    I = zeros(n,n)
    for i in 1:n
        I[i,i] =1
    end
    for i in 1:(n-1)
        for j in (i+1):n
            m = x[j,i]/x[i,i]
            x[j,i:end] = x[j,i:end].-x[i,i:end].*m
            I[j,:] = I[j,:].-I[i,:].*m
        end
    end
    for i in n:-1:2
        for j in (i-1):-1:1
            m = x[j,i]/x[i,i]
            x[j,i]=0
            I[j,:] =I[j,:] .- I[i,:].*m
        end
    end
    for i in 1:n
        I[i,:] = I[i,:]./x[i,i]
    end
    return I
end

function Fsolve_Gaussian(x,b)
    n=length(b)
    x = reshape(x,n,n)
    for i in 1:(n-1)
        for j in (i+1):n
            m = x[j,i]/x[i,i]
            x[j,i:end] = x[j,i:end].-x[i,i:end].*m
            b[j] = b[j]-b[i]*m
        end
    end
    return b
end

Fsolve_Gaussian([1 2 3; 4 5 6;7 8 9],[1;2;3])
[1 2 3; 4 5 6;7 8 9]*[1 ,-2, 0]

#Finverse_Gaussian([1 2;6 7])

#[1 2;6 7]*Finverse_Gaussian([1 2;6 7])

# safer version of Gaussian solve -judge and shift terms with 0

#function Fsolve_Gaussian_safely(x,b)
#    n=length(b)
#    for j in 1:(n-1)
#    end
#end

x = [1 2 3;4 5 6;7 8 9]

function LU(x)
    n=Int(sqrt(length(x)))
    for j in 1:(n-1)
        for i in (j+1):n
            m = x[i,j] / x[j,j]
            x[i,(j+1):n] = x[i,(j+1):n].-m.*x[j,(j+1):n]
            x[i,j] = m
        end
    end
    return x
end
LU(x)


function bicgstab(a::Matrix{Float64}, b::Vector{Float64}, x0::Vector{Float64};
  tol=1.0e-10, limit=1000, trace=false)

  n = size(a, 1)
  x = copy(x0)
  local converged

  r = b - a * x
  r0_hat = copy(r)
  rho0 = w = α = 1.0
  v = zeros(n)
  p = zeros(n)
  rho1 = dot(r0_hat, r)
  trace && println("\nFirst few iterations:")
  iters = 0
  while true
    iters += 1
    converged = norm(r) < tol * norm(b)
    (converged || (iters == limit)) && break
    β = (rho1 / rho0) * (α / w)
    p = r + β * (p - w * v)
    v = a * p
    α = rho1 / dot(r0_hat, v)
    s = r - α * v
    t = a * s
    w = dot(t, s) / dot(t, t)
    rho0 = rho1
    rho1 = -w * dot(r0_hat, t)
    x += α * p + w * s
    r = s - w * t
    if trace && iters <= 5
      println("After $iters iterations: x = $x")
    end
  end
  iters, converged, x
end


#=
  Original Author: Anastasia Yendiki

  Copyright © 2022 The General Hospital Corporation (Boston, MA) "MGH"

  Terms and conditions for use, reproduction, distribution and contribution
  are found in the 'FreeSurfer Software License Agreement' contained
  in the file 'LICENSE' found in the FreeSurfer distribution, and here:

  https://surfer.nmr.mgh.harvard.edu/fswiki/FreeSurferSoftwareLicense

  Reporting: freesurfer@nmr.mgh.harvard.edu
=#

export cart2pol, pol2cart, cart2sph, sph2cart

"""
    cart2pol(x, y)

    Transform Cartesian coordinates (`x`, `y`) to polar coordinates (`φ`, `ρ`),
    where `φ` is in radians.
"""
function cart2pol(x, y)

  φ = atan.(y, x)
  ρ = hypot.(x, y)

  return φ, ρ
end

"""
    pol2cart(φ, ρ)

    Transform polar coordinates (`φ`, `ρ`) to Cartesian coordinates (`x`, `y`),
    where `φ` is in radians.
"""
function pol2cart(φ, ρ)

  x = ρ .* cos.(φ)
  y = ρ .* sin.(φ)

  return x, y
end

"""
    cart2sph(x, y, z)

    Transform Cartesian coordinates (`x`, `y`, `z`) to spherical coordinates
    (`φ`, `θ`, `ρ`), where `φ` and `θ` are in radians.
"""
function cart2sph(x, y, z)

  hypotxy = hypot.(x, y)
  ρ = hypot.(hypotxy, z)
  θ = atan.(z, hypotxy)
  φ = atan.(y, x)

  return φ, θ, ρ
end

"""
    sph2cart(φ, θ, ρ)

    Transform spherical coordinates (`φ`, `θ`, `ρ`) to Cartesian coordinates
    (`x`, `y`, `z`).

    `φ` and `θ` must be in radians.
"""
function sph2cart(φ, θ, ρ)

  z = ρ .* sin.(θ)
  ρcosθ = ρ .* cos.(θ)
  x = ρcosθ .* cos.(φ)
  y = ρcosθ .* sin.(φ)

  return x, y, z
end

"""
    ang2rot(φ, θ)

    Convert polar `φ` and azimuthal `θ` angles to a rotation matrix.

    `φ` and `θ` must be in radians.
"""
function ang2rot(φ, θ)

  # Rotate by φ around the z-axis
  Rz = [cos(φ)  -sin(φ)  0
        sin(φ)   cos(φ)  0
        0         0      1]

  # Rotate by θ around the y-axis
  Ry = [cos(θ)   0   sin(θ)
        0        1   0
       -sin(θ)   0   cos(θ)]

  R =  Rz * Ry

  return R
end


function subbac!(u::Matrix{Float64}, b::Vector{Float64})
# Back-substitution on an Upper Triangle
  n = size(u, 1)
  for i in n:-1:1
    total = b[i]
    if i < n
      for j in i+1:n
        total -= u[i, j] * b[j]
      end
    end
    b[i] = total / u[i, i]
  end
end

function subbac(u::Matrix{Float64}, b::Vector{Float64})
# Back-substitution on an Upper Triangle
  bt = deepcopy(b)
  n = size(u, 1)
  for i in n:-1:1
    total = bt[i]
    if i < n
      for j in i+1:n
        total -= u[i, j] * bt[j]
      end
    end
    bt[i] = total / u[i, i]
  end
  bt
end

export
  subbac!,
  subbac


using FiniteDiff, LinearAlgebra, SparseArrays, Test, StaticArrays

function f(x)
  xm1 = [0;x[1:end-1]]
  xp1 = [x[2:end];0]
  xm1 - 2x + xp1
end

function second_derivative_stencil(N)
  A = zeros(N,N)
  for i in 1:N, j in 1:N
      (j-i==-1 || j-i==1) && (A[i,j]=1)
      j-i==0 && (A[i,j]=-2)
  end
  A
end

x = @SVector ones(30)
J = FiniteDiff.finite_difference_jacobian(f, x, Val{:forward}, eltype(x))
@test J ≈ second_derivative_stencil(30)

J = FiniteDiff.finite_difference_jacobian(f, x, Val{:central}, eltype(x))
@test J ≈ second_derivative_stencil(30)

J = FiniteDiff.finite_difference_jacobian(f, x, Val{:complex}, eltype(x))
@test J ≈ second_derivative_stencil(30)

spJ = sparse(second_derivative_stencil(30))
J = FiniteDiff.finite_difference_jacobian(f, x, Val{:forward}, eltype(x), jac_prototype=spJ)
@test J ≈ second_derivative_stencil(30)
@test typeof(J) == typeof(spJ)
J = FiniteDiff.finite_difference_jacobian(f, x, Val{:forward}, eltype(x),
    colorvec=SVector{30}(repeat(1:3, 10)), sparsity=spJ, jac_prototype=spJ)
@test J ≈ second_derivative_stencil(30)
@test typeof(J) == typeof(spJ)

f(x) = x
@testset "1x1 test of $x" for
  (x, y) in ((SVector{1}([1.]), SMatrix{1,1}), ([1.0], Matrix)),
    difftype in (:forward, :central, :complex)

  J = FiniteDiff.finite_difference_jacobian(f, x, Val{difftype}, eltype(x))
  @test J[1, 1] ≈ 1.0
  @test J isa y
end

x = SVector{1}([1.])
f(x) = vcat(x, x)
J = FiniteDiff.finite_difference_jacobian(f, x, Val{:forward}, eltype(x))
@test J ≈ fill(1.0, 2, 1)
@test J isa SMatrix{2,1}
J = FiniteDiff.finite_difference_jacobian(f, x, Val{:central}, eltype(x))
@test J ≈ fill(1.0, 2, 1)
@test J isa SMatrix{2,1}
J = FiniteDiff.finite_difference_jacobian(f, x, Val{:complex}, eltype(x))
@test J ≈ fill(1.0, 2, 1)
@test J isa SMatrix{2,1}


"""
    pearson_correlation(x, y)

Find the pearson correlation between two variables.

# Example:

julia> PearsonCorrelation([12,11,16,17,19,21],[11,51,66,72,12,15])
-0.2092706263573845

Contribution by: [Aru Bhardwaj](https://github.com/arubhardwaj)

"""

function pearson_correlation(x, y)
    mean_x = sum(x) / length(x)
    mean_y = sum(y) / length(y)
    XY = (x .- mean_x) .* (y .- mean_y)
    XXs = sum((x .- mean_x) .* (x .- mean_x))
    YYs = sum((y .- mean_y) .* (y .- mean_y))
    return(sum(XY) / (sqrt(XXs .* YYs)))
end


# Monte-Carlo Applications

# Type 1
# path like integration
# ∫₀¹ e
# todo

# Type 2
# Volume-like integration

# E.g. evaluate π

# 1. generate n points within a unit □
#    enclosed by xlim<-[0 1]
#    and         ylim<-[0 1]

n = 1000
x = rand(n)
y = rand(n)

# 2. Judge # of pts in the unit ◷
s = 0
for i in 1:n# I hate for loop , there's a simpler version below
    if (x[i]^2 + y[i]^2)<=1
        global s+=1
    end
end
s

# Area of ▧ / Area of ◍  = 4/π
π_est = 4s/n
π_est

# Vectorize? replace 2 with
#even faster?
@time size(zeros(n)[x.^2+y.^2 .<=1])
#0.000070 seconds (27 allocations: 42.813 KiB)
#Compared with

s=0
@time for i in 1:n
    if (x[i]^2 + y[i]^2)<=1
        global s+=1
    end
end
#0.000220 seconds (7.75 k allocations: 136.734 KiB)

π_est = 4s/n
π_est

plot(x,y;seriestype="scatter")
plot!(0:.01:1,sqrt.(1 .-(0:.01:1).^2))


# determine the moment of the stoichiometric matrix coefficient
# necessary to deal with (independent) stochastic variables
# NOTE: only geometric distribution is supported
# the parameter passed is supposed to be the MEAN value
#function expected_coeff(x::Union{Int, Sym{ModelingToolkit.Parameter{Real}}}, pwr)
function expected_coeff(x::Union{Int, Symbolic}, pwr)
    if typeof(x) == Int64
        x^pwr
    else
        geometric_raw_moment(x, pwr)
    end
end

function eulerian_number(n, k)
    suma = 0
    for j in 0:k+1
        suma += (-1)^j * binomial(n+1, j) * (k-j+1)^n
    end
    return suma
end

#function geometric_raw_moment_arbitrary_order(m::Sym{ModelingToolkit.Parameter{Real}}, n::Int)
function geometric_raw_moment_arbitrary_order(m::Symbolic, n::Int)
    # computes the symbolic expression of ⟨Xⁿ⟩ where X is a stochastic variable
    # following a geometric distribution with mean burst size m
    # the distribution has the form P(x) = (1-p)ˣp, and m is given by m = (1-p)/p
    #@syms p
    suma = 0.0
    for i in 0:n
        #suma += eulerian_number(n, i) * (1-p)^(n-i)
        suma += eulerian_number(n, i) * m^(n-i) * (1+m)^(i-n)
        suma = simplify(suma)
    end
    #return p^(-n) * suma
    simplify((1+m)^n * suma)
end

#function geometric_raw_moment(m::Sym{ModelingToolkit.Parameter{Real}}, n::Int)
function geometric_raw_moment(m::Symbolic, n::Int)
    # provide the symbolic expression of ⟨Xⁿ⟩ where X is a stochastic variable
    # following a geometric distribution with mean burst size m
    # the distribution has the form P(x) = (1-p)ˣp, and m is given by m = (1-p)/p
    # Although geometric_raw_moment_arbitrary_order() can obtain the moment up to
    # arbitrary order, it is unable to format it in a nice and readable way
    # (not straightforward to use SymbolicUtils.jl to perform a binomial expansion
    # and hence tidy up the expression). For this reason, we provide the clean
    # symbolic expressions up to order 5 just to make the moment equations easier to check
    if n == 0
        1
    elseif n==1
        m
    elseif n==2
        2*m^2+m
    elseif n==3
        m*(6*m^2 + 6*m + 1)
    elseif n==4
        m*(2*m + 1)*(12*m^2+12*m+1)
    elseif n==5
        m*(120*m^4 + 240*m^3 + 150*m^2 + 30*m + 1)
    elseif n>5
        geometric_raw_moment_arbitrary_order(m, n)
    else
        error("negative argument values")
    end
end


# Richardson function
function numdiff(f,x::Number,h::Float64)
    D0 = (f(x+h)-f(x-h))/(2*h);
    D1 = ( f(x+(h/2)) - f(x-(h/2)) )/(h);
    TV = D1 + (D1-D0)/3
    return TV
end

# Richardson function
function numdiff(f,x::Number,h::Int64)
    D0 = (f(x+h)-f(x-h))/(2*h);
    D1 = ( f(x+(h/2)) - f(x-(h/2)) )/(h);
    TV = D1 + (D1-D0)/3
    return TV
end

# Richardson discrete of type 1 (We get an exact value in x vector in which we want to differentiate)
function numdiff(x::Array{Float64,1}, y::Array{Float64,1}, val) #We get a x array, a y array and a val in x for which we want to get its derivative
    h = abs(x[2] - x[1]);
    ind = findall(p->p==val, x)[1]
    D0 = (y[ind+2]-y[ind-2])/(4*h);
    D1 = (y[ind+1]-y[ind-1])/(2*h);
    TV = D1 + (D1-D0)/3
    return TV
end

# Richardson getting the whole bunch i.e. CDD of O(h^4) residues

#=
function numdiff(x::Array{Float64,1}, y::Array{Float64,1}) #We get a x array, a y array and a val in x for which we want to get its derivative
    h = abs(x[2] - x[1]);

    return TV
end
=#

# Forward func of linear order
function fdiff(f,x,h)
    d = (f(x+h)-f(x))/h;
    return d
end

# Forward discrete of type 1 (We get an exact value in x vector in which we want to differentiate)
function fdiff(x::Array{Float64,1}, y::Array{Float64,1},val)
    h = abs( x[2] - x[1] );
    ind = findall(p->p==val, x)[1]
    d = (y[ind+1] - y[ind])/h
    return d
end

# Forward discrete, with the whole bunch (MATLAB-like, with the whole vector)
function fdiff(x::Array{Float64,1}, y::Array{Float64,1})
    h = abs( x[2] - x[1] );
    d = (y[2:end] - y[1:end-1])/h
    return d
end

# Backwards func
function bdiff(f,x,h)
    d = (f(x)-f(x-h))/h;
    return d
end

# Backwards discrete of type 1 (We get an exact value in x vector in which we want to differentiate)
function bdiff(x::Array{Float64,1}, y::Array{Float64,1},val)
    h = abs( x[2] - x[1] );
    ind = findall(p->p==val, x)[1]
    d = (y[ind] - y[ind-1])/h
    return d
end

# Backwards discrete, with the whole bunch (MATLAB-like, with the whole vector)
function bdiff(x::Array{Float64,1}, y::Array{Float64,1})
    h = abs( x[2] - x[1] );
    d = (y[2:end] - y[1:end-1])/h
    return d
end


using ModelingToolkit, OrdinaryDiffEq, Test, LinearAlgebra

@parameters t σ ρ β
@variables x(t) y(t) z(t) k(t)
D = Differential(t)

eqs = [D(D(x)) ~ σ*(y-x),
       D(y) ~ x*(ρ-z)-y,
       D(z) ~ x*y - β*z]

sys′ = ODESystem(eqs)
sys = ode_order_lowering(sys′)

eqs2 = [0 ~ x*y - k,
        D(D(x)) ~ σ*(y-x),
        D(y) ~ x*(ρ-z)-y,
        D(z) ~ x*y - β*z]
sys2 = ODESystem(eqs2, t, [x, y, z, k], parameters(sys′))
sys2 = ode_order_lowering(sys2)
# test equation/varible ordering
ModelingToolkit.calculate_massmatrix(sys2) == Diagonal([1, 1, 1, 1, 0])

u0 = [D(x) => 2.0,
      x => 1.0,
      y => 0.0,
      z => 0.0]

p  = [σ => 28.0,
      ρ => 10.0,
      β => 8/3]

tspan = (0.0,100.0)
prob = ODEProblem(sys,u0,tspan,p,jac=true)
probexpr = ODEProblemExpr(sys,u0,tspan,p,jac=true)
sol = solve(prob,Tsit5())
solexpr = solve(eval(prob),Tsit5())
@test all(x->x==0,Array(sol - solexpr))
#using Plots; plot(sol,vars=(:x,:y))

@parameters t σ ρ β
@variables x(t) y(t) z(t)
D = Differential(t)

eqs = [D(x) ~ σ*(y-x),
       D(y) ~ x*(ρ-z)-y,
       D(z) ~ x*y - β*z]

lorenz1 = ODESystem(eqs,name=:lorenz1)
lorenz2 = ODESystem(eqs,name=:lorenz2)

@variables α(t)
@parameters γ
connections = [0 ~ lorenz1.x + lorenz2.y + α*γ]
connected = ODESystem(connections,t,[α],[γ],systems=[lorenz1,lorenz2])

u0 = [lorenz1.x => 1.0,
      lorenz1.y => 0.0,
      lorenz1.z => 0.0,
      lorenz2.x => 0.0,
      lorenz2.y => 1.0,
      lorenz2.z => 0.0,
      α => 2.0]

p  = [lorenz1.σ => 10.0,
      lorenz1.ρ => 28.0,
      lorenz1.β => 8/3,
      lorenz2.σ => 10.0,
      lorenz2.ρ => 28.0,
      lorenz2.β => 8/3,
      γ => 2.0]

tspan = (0.0,100.0)
prob = ODEProblem(connected,u0,tspan,p)
sol = solve(prob,Rodas5())
@test maximum(sol[2,:] + sol[6,:] + 2sol[1,:]) < 1e-12
#using Plots; plot(sol,vars=(:α,Symbol(lorenz1.x),Symbol(lorenz2.y)))


using LinearAlgebra
using Statistics

x = [ 2, -1, 2];

# norm of x is written in Julia as norm(x)
norm(x)

sqrt(x'*x)

sqrt(sum(x.^2))

x = randn(10); y = randn(10)

lhs = norm(x+y)

rhs = norm(x) + norm(y)

# rms value of a vector is rms(x) = norm(x)/sqrt(length(x))
rms(x) = norm(x) / sqrt(length(x))

t = 0:0.01:1; # list of times

x = cos.(8*t) - 2*sin.(11*t)

rms(x)

using Plots

plot(t,x)

plot!(t, avg(x)*ones(length(x)))

plot!(t, (avg(x)+rms(x))*ones(length(x)), color = :green)

plot!(t, (avg(x)-rms(x))*ones(length(x)), color = :green)

plot!(legend = false)

# Define Chebyshev bound function
# The Chebyshev inequality states that the number of entries of an
# n-vector x that have absolute value at least a is no more than
# ∥x∥2/a2 = nrms(x)2/a2. If this number is, say, 12.15,
# we can conclude that no more that 12 entries have absolute value
# at least a, since the number of entries is an integer.

cheb_bound(x,a) = floor(norm(x)^2/a);
a = 1.5;
cheb_bound(x,a)
# Number of entries of x with |x_i| >= a
sum(abs.(x) .>= a)


# Algorithm 5.3
# Runge Kutta Fehlberg

function rkf(f, a, b, α, max_step, min_step, tolerance)
	flag = 1
	result = Tuple{Float64, Float64, Float64}[]
	t = a
	k = Array{Float64}(undef, 6)
	h = max_step
	ω = α
	push!(result, (t, ω, h))
	while flag == 1
		k[1] = h*f(t, ω)
		k[2] = h*f(t + (1/4)*h, ω + (1/4)*k[1])
		k[3] = h*f(t + (3/8)*h, ω + (3/32)*k[1] + (9/32)*k[2])
		k[4] = h*f(t + (12/13)*h, ω + (1932/2197)*k[1] - (7200/2197)*k[2] + (7296/2197)*k[3])
		k[5] = h*f(t + h, ω + (439/216)*k[1] - 8*k[2] + (3680/513)*k[3] - (845/4104)*k[4])
		k[6] = h*f(t + (1/2)*h, ω - (8/27)*k[1] + 2*k[2] - (3544/2565)*k[3] + (1859/4104)*k[4] - (11/40)*k[5])
		R = (1/h)*abs((1/360)*k[1] - (128/4275)*k[3] - (2197/75240)*k[4] + (1/50)*k[5] + (2/55)k[6])
		if R <= tolerance
			t = t + h
			ω = ω + (25/216)*k[1] + (1408/25655)*k[3] + (21977/4104)*k[4] - (1/5)*k[5]
			push!(result, (t, ω, h))
		end
		δ = 0.84 * (tolerance/R)^(1/4)
		if δ <= 0.1
			h = 0.1*h
		elseif δ >= 4
			h = 4h
		else
			h = δ*h
		end
		if h > max_step
			h = max_step
		end
		if t >= b
			flag = 0
		elseif (t + h) > b
			h = b - t
		elseif h < min_step
			flag = 0
		end
	end
	return result
end

f(t, y) = y - t^2 + 1
println(rkf(f, 0, 2, 0.5, 0.25, 0, 10^(-5)))


module NewtonMethod2

using LinearAlgebra, Statistics, ForwardDiff

function newtonroot(f, f_prime; x0, tol=1E-7, maxiter=1000)
    # setup the algorithm
    x_old = x0
    normdiff = Inf
    iter = 0
    while normdiff > tol && iter <= maxiter
        x_new = x_old-f(x_old)/f_prime(x_old) # use the passed in map
        normdiff = norm(x_new - x_old)
        x_old = x_new
        iter = iter + 1
    end
    if iter <= maxiter
        return (x_old, normdiff, iter)
    else
        return (nothing, normdiff, iter) # hope it is correct!
    end
end

export newtonroot

#########

function newtonroot2(f; x0, tol=1E-7, maxiter=1000)
    D(f) = x -> ForwardDiff.derivative(f, x)
    f_prime = D(f)
    # setup the algorithm
    x_old = x0
    normdiff = Inf
    iter = 0
    while normdiff > tol && iter <= maxiter
        x_new = x_old-f(x_old)/f_prime(x_old) # use the passed in map
        normdiff = norm(x_new - x_old)
        x_old = x_new
        iter = iter + 1
    end
    if iter <= maxiter
        return (x_old, normdiff, iter)
    else
        return (nothing, normdiff, iter)
    end
end

export(newtonroot2)

end


### A Pluto.jl notebook ###
# v0.17.4

using Markdown
using InteractiveUtils

# ╔═╡ d16725a0-6cd3-11ec-0998-fd4af26f10dd
begin
	using Pkg; Pkg.activate(@__DIR__); Pkg.instantiate()
	Pkg.precompile()

	using PlutoUI
	using Plots
	using Statistics
	using StatsBase
	using PyCall
	using DataFrames
	using GLM
	using Tables
	using XLSX
	using DataStructures
	using MLBase
	using RDatasets
	using LsqFit
end;

# ╔═╡ 91471950-703f-4927-8bce-f4478f4a53ba
PlutoUI.TableOfContents(aside=true, indent=true, depth=3)

# ╔═╡ a53d44c5-75c4-4508-aa19-33e838c9e0bb
md"""
# 📈 Regression

Regression Analysis is a set of statistical processes for estimating the relationships between a _dependent variable_ (often called the _outcome_ or _response_ variable) and one or more _independent variables_ (often called _predictors_, _covariates_, _explanatory variables_ or _features_). The most common form of regression analysis is linear regression, in which one finds the line (or a more complex linear combination) that most closely fits the data according to a specific mathematical criterion.

[YouTube Link](https://www.youtube.com/watch?v=5TCbIK_cpZE)
"""

# ╔═╡ e9458f99-4ec0-4167-b771-3090dfec6320
md"""
## Simple linear regression
"""

# ╔═╡ 784c5b0e-59dc-4b2d-bf40-112261ac8e85
md"""
### Building from scratch

Firstly, let's generate some random data and then plot it...
"""

# ╔═╡ 65e22edc-ef5c-412f-82f4-1c84da209d6d
begin
	# x values
	xvals = repeat(1:0.5:10, inner=2)
	# y values (with some noise)
	yvals = 3 .+ xvals .+ 2 .* rand(length(xvals)) .-1

	# plotting
	scatter(xvals, yvals, color=:black, leg=false,
			xlabel="X values", ylabel="Y values")
end

# ╔═╡ db2c3459-2127-4b46-8e73-546850767dc0
md"""
Assuming an ordinary least squares regression, the slope `a` of the fitted line is equal to the correlation between `y` and `x` corrected by the ratio of standard deviations of these variables. The intercept `b` of the fitted line is such that the line passes through the center of mass (`x`, `y`) of the data points.

Thus, we can easily build the `find_best_fit` function, which returns `a` and `b` coefficients.
"""

# ╔═╡ 3bb8c382-f97d-48dc-8cbe-99bf77de0784
function find_best_fit(x, y)
	X̅, Y̅ = mean(x), mean(y)
	Sˣ, Sʸ = std(x), std(y)

	a = cor(x,y) * (Sˣ/Sʸ)
	b = Y̅ - a*X̅

	return a, b
end

# ╔═╡ 75059ff4-6b9d-4e01-9aed-33ea8b0e9a71
md"""
We can now use the function above to calculate `a` and `b` and then compute the estimated values `ŷvals`...
"""

# ╔═╡ e83331a1-8655-4de0-a36f-b74b5e43674b
begin
	a, b = find_best_fit(xvals, yvals)
	ŷvals₁ = a .* xvals .+ b
end

# ╔═╡ d81e448b-b4e4-4d6f-8ca7-42c41569977d
md"""
### numpy

We can also import the `numpy` python package to perform a simple linear regression. We are able to use any `numpy` function by adding the prefix `np`, just like in Python.
"""

# ╔═╡ a047abfb-178c-40ac-8f8c-c3a8a73c1849
# importing numpy
np = pyimport("numpy");

# ╔═╡ 208af138-5131-4034-885b-c41f6ccf36f5
# numpy model
np_model = np.polyfit(xvals, yvals, 1)

# ╔═╡ 39884fc4-c41c-46e5-a92a-44d766243715
# estimated ŷ values
ŷvals₂ = np_model[1] .* collect(xvals) .+ np_model[2]

# ╔═╡ 13719feb-60ae-433b-bf1e-df6ac045215d
md"""
### GLM

Finally, we can use the [GLM.jl](https://github.com/JuliaStats/GLM.jl) package to perform simple linear regression using the `lm` function.
"""

# ╔═╡ 10c072d4-b793-458c-b947-d5dc3aae5e25
# converting data into a DataFrame
data = DataFrame(X=xvals, Y=yvals);

# ╔═╡ 68a0574f-5b17-48ae-b8c4-a03c7e385561
# GLM model
glm_model = lm(@formula(Y ~ X), data)

# ╔═╡ ecb1cb29-2b57-4197-9327-e762fc876281
# estimated ŷ values
ŷvals₃ = predict(glm_model)

# ╔═╡ 056b28f3-8da4-4a94-a33c-72b30c8aa0aa
md"""
### Comparing methods

Now we can plot all the three models. We can see below that both `numpy` and `GLM` models are exactly the same. The model that we build from scratch is slightly different.
"""

# ╔═╡ 9a5dcad1-9365-423c-b27a-a6c206850848
begin
	# plotting data
	scatter(xvals, yvals, color=:black, label="Data",
			xlabel="X values", ylabel="Y values", legend=:topleft)

	# plotting "from scratch" fitted line
	plot!(xvals, ŷvals₁, color=:red, label="From Scratch")

	# plotting numpy model
	plot!(xvals, ŷvals₂, color=:green, label="Numpy")

	# plotting GLM model
	plot!(xvals, ŷvals₃, color=:blue, label="GLM")
end

# ╔═╡ a89ec73a-5e35-49ad-8829-dc2ecc7e8aac
md"""
## Load real data

Now let's get some real data. We will use housing information from zillow, check out the file `zillow_data_download_april2020.xlsx` for a quick look of what the data looks like. Our goal will be to build a linear regression model between the number of houses listed vs the number of houses sold in a few states. Fitting these models can serve as a key real estate indicator.
"""

# ╔═╡ aa78c706-8860-46f1-9982-ea10bdc4fccb
# loading data
S = XLSX.readxlsx("data/zillow_data_download_april2020.xlsx")

# ╔═╡ 618ab9ee-ee9c-4dbb-b48f-ebeeb15549c6
md"""
**Note:** there are three different sheets in our data.

## Data wrangling

Let's create a `DataFrame` from the `Sale_counts_city` sheet...
"""

# ╔═╡ b530e842-bfa4-4c41-9cc9-4aa629e12f99
# selecting sheet
sale_counts = S["Sale_counts_city"][:]

# ╔═╡ 041e87c7-deee-46c1-85be-ea47df16217c
# converting Matrix into DataFrame
df_sale_counts = DataFrame(sale_counts[2:end,:], Symbol.(sale_counts[1,:]))

# ╔═╡ 19a44596-2367-4964-b21e-7d84ba147915
md"""
Now, we can do the same with the `MonthlyListings_City` sheet...
"""

# ╔═╡ ab8cf8d2-9ac6-4361-b168-243f781e7dfa
begin
	# selecting sheet
	month_list = S["MonthlyListings_City"][:]
	# converting Matrix into DataFrame
	df_month_list = DataFrame(month_list[2:end,:], Symbol.(month_list[1,:]))
end

# ╔═╡ 215483a1-6059-437d-a4b6-ca3cd897f13b
md"""
We may filter just the data from February 2020...
"""

# ╔═╡ b075d090-91aa-4d1f-bb9f-a93dd65413e7
begin
	# monthly listing data (Feb 2020)
	monthlist_2020_2 = df_month_list[!,[1,2,3,4,5,end]]
	rename!(monthlist_2020_2, Symbol("2020-02") => :listing)
end

# ╔═╡ 1b8f7b0d-a843-4af7-90b7-4a64d74edab0
begin
	# sale counts data (Feb 2020)
	salecounts_2020_2 = df_sale_counts[!,[1,end]]
	rename!(salecounts_2020_2, Symbol("2020-02") => :sales)
end

# ╔═╡ 39456bea-24dc-482b-9d8b-33a9f2ed715a
md"""
Next, we can (outer) join our two DataFrames and, then, remove missing values...
"""

# ╔═╡ f195db71-f2a4-4f5c-bfa4-2d9ec1f23a32
begin
	feb2020data = outerjoin(monthlist_2020_2, salecounts_2020_2, on=:RegionID)
	dropmissing!(feb2020data)
end

# ╔═╡ 2da58c4e-8db4-4d80-aa29-f3ec10952f73
md"""
Let's define variables for all the three columns we are interested in...
"""

# ╔═╡ 93179d9a-5bf4-488b-b710-28a5f56b2e26
begin
	sales = feb2020data[!,:sales]
	counts = feb2020data[!,:listing]
	states = feb2020data[!,:StateName]
end;

# ╔═╡ 560a134f-1175-4d56-83af-1ad6bbe1a99f
md"""
Now, we can select just the top 10 states (most cited states). We will use the `counter` function from the [DataStructures.jl](https://github.com/JuliaCollections/DataStructures.jl) package.
"""

# ╔═╡ 0c4a00f6-69f2-4fc3-a7f8-ca6e38404ac9
# counter object (Accumulator)
C = counter(states)

# ╔═╡ a02a9fdf-46bc-4ea1-bb29-b1731a66c057
# converting into a Dictionary
Cdict = C.map

# ╔═╡ 63b596be-9392-4efb-8d31-7f1d6e9c7254
begin
	# getting counts as an Array
	countvals = values(Cdict) |> collect

	# getting state names as an Array
	state_labels = keys(Cdict) |> collect
end;

# ╔═╡ feced955-2e69-44ae-bc2a-82d4f2136b38
# getting indexes from 10 top states
topstateindexes = sortperm(countvals, rev=true)[1:10]

# ╔═╡ 831484f5-6cdc-4641-8adf-117a33c518c2
md"""
**Note:** the built-in `sortperm` function returns a permutation vector `I` that puts `v[I]` in sorted order. The order is specified using the same keywords as `sort!`.
"""

# ╔═╡ b94019d8-a7d8-4781-819b-28eeac823812
# getting top 10 states names
topstates = state_labels[topstateindexes]

# ╔═╡ 4735fc87-68cb-4617-aa7c-6139ba7b7c0a
md"""
## Linear regression models

Firstly, we will plot the model for each state using the formula `Y ~ X`.
"""

# ╔═╡ ec7ae9e6-b191-4578-8a47-9d2245280316
begin
	# creating a blank Array of Plot
	all_plots₁ = Array{Plots.Plot,1}(undef,10)

	for (i,s) in enumerate(topstates)
		# finding indexes of all occurences of a given state
		ids = findall(feb2020data[!,:StateName] .== s)
		# getting data for a given state
		data = DataFrame(X=Float64.(counts[ids]), Y=Float64.(sales[ids]))
		# training the model
		ols = GLM.lm(@formula(Y ~ X), data)

		# plotting data by each state
		all_plots₁[i] = scatter(data.X, data.Y, markersize=2,
								xlims=(0,500), ylims=(0,500),
								color=i, aspect_ratio=:equal,
								legend=false, title=s)

		# plotting model by each state
		plot!(data.X, predict(ols), color=:black)
	end

	# display plots
	plot(all_plots₁..., layout=(2,5), size=(850,300))
end

# ╔═╡ 216dc17e-54c1-4520-890d-8663165b10dd
md"""
We can do the same thing, but now we will plot all the elements within the same figure.

**Note:** when we use the `Y ~ X + 0` formula, it forces the intercept to be zero.
"""

# ╔═╡ dddfb8fb-7047-4417-8dd2-569a93277f0c
begin
	plot()

	for (i,s) in enumerate(topstates)
	    ids = findall(feb2020data[!,:StateName] .== s)
	    data = DataFrame(X=Float64.(counts[ids]), Y=Float64.(sales[ids]))
	    ols = GLM.lm(@formula(Y ~ 0 + X), data)

	    scatter!(counts[ids], sales[ids], markersize=2, xlim=(0,500),
				 ylim=(0,500), color=i, aspect_ratio=:equal,
	        	 legend=false, marker=(3,3,stroke(0)), alpha=0.2)

	    if s ∈ ["NC","CA","FL"]
	        annotate!([(500-20,10+coef(ols)[1]*500, text(s,10))])
	    end

	    plot!(counts[ids], predict(ols), color=i, linewidth=2)
	end

	xlabel!("Listings")
	ylabel!("Sales")
end

# ╔═╡ 5116c117-75e4-4989-b06f-2883bba48330
md"""
## Logistic regression

So far, we have shown several ways to solve the linear regression problem in Julia. Here, we will first start with a motivating example of when you would want to use logistic regression. Let's assume that our predictor vector is binary (`0` or `1`), let's fit a linear regression model.
"""

# ╔═╡ 3a86069b-3f24-4eec-a7ae-f77189a4fd77
# generating random data
𝒮 = DataFrame(X=collect(1:7), Y=[1,0,1,1,1,1,1])

# ╔═╡ 453385dd-8bda-42b8-9b4a-6a4963b21346
# linear regression model
lin_reg = lm(@formula(Y ~ X), 𝒮)

# ╔═╡ bd31693d-dee9-4534-b4d4-3904997e8020
begin
	# plotting data
	scatter(𝒮.X, 𝒮.Y, legend=false, xlabel="X", ylabel="Y")
	#plotting model
	plot!(1:7, predict(lin_reg))
end

# ╔═╡ 0fb142cd-ac45-42e8-b5d1-24c6142392bb
md"""
What this plot quickly shows is that linear regression may end up predicting values outside the `[0,1]` interval. For an example like this, we will use logistic regression. Interestingly, a generalized linear model (https://en.wikipedia.org/wiki/Generalized_linear_model) unifies concepts like linear regression and logistic regression, and the [GLM.jl](https://github.com/JuliaStats/GLM.jl) package allows you to apply either of these regressions easily by specifying the `distribution family` and the `link` function.

To apply logistic regression via the `GLM.jl` package, you can readily use the `Binomial` family and the `LogitLink` link function.

### Load data

Firstly, let's load some data from the [RDatasets.jl](https://github.com/JuliaStats/RDatasets.jl) package...
"""

# ╔═╡ 3caa5331-2507-4299-b2e4-50e8a5cbbe46
cats = dataset("MASS", "cats")

# ╔═╡ 22c201af-9437-4dde-8653-508ebf514008
md"""
### Data wrangling

We will map the sex of each cat to a binary `0/1` value.
"""

# ╔═╡ 16367b7a-65ea-4fa0-abad-a1cc25c60ee6
# getting the map
sex_map = labelmap(cats.Sex)

# ╔═╡ 1d7ef9db-dd15-4ed4-841a-b4baf02302a1
# encoding Sex column
sex_enconde = labelencode(sex_map, cats.Sex)

# ╔═╡ f42bdc3a-943b-4157-853d-860af52e059c
# plotting data by Sex
scatter(cats.BWt, cats.HWt, color=sex_enconde, legend=false)

# ╔═╡ 310f2282-d6dd-405d-8399-e75fdc75c27c
md"""
Females (blue) seem to be more present in the lower left corner and Males (orange) seem to be present in the top right corner.

We will use the `HWt` variable as the only feature and the `encoded_sex` as the target. Let's convert labels from `1/2` into `0/1`...
"""

# ╔═╡ cc911a53-9139-4c90-b02b-15f30e6772f0
cats_df = DataFrame(X=cats[!,:HWt], Y=sex_enconde .- 1)

# ╔═╡ 79bbe213-c15a-4ecb-aef7-107846234f1d
md"""
### Modeling data

Let's run a logistic regression model on this data.
"""

# ╔═╡ 11b1f5d1-7ae2-43dd-bd35-5b0ba6fde6f4
# logistic model
probit = glm(@formula(Y ~ X), cats_df, Binomial(), LogitLink())

# ╔═╡ e3e9d570-af57-4065-84ec-b61400fa449d
begin
	# plotting ground truth data
	scatter(cats_df.X, cats_df.Y, label="Ground Truth", color=6)

	# plotting predictions
	scatter!(cats_df.X, predict(probit), label="Predictions", color=7,
			 xlabel="Heart Rate", ylabel="Pr(Sex=M)")
end

# ╔═╡ dfc76a4c-4e2b-4588-92e1-2209507fd670
md"""
**Note:** as you can see, contrary to the linear regression case, the predicted values do not go beyond `1`.
"""

# ╔═╡ 31308555-dfc9-4e28-9108-8a83d65b7770
md"""
## Non-linear regression

Finally, sometimes you may have a set of points and the goal is to fit a non-linear function (maybe a quadratic function, a cubic function, an exponential function...). The way we would solve such a problem is by minimizing the least square error between the fitted function and the observations we have. We will use the package [LsqFit.jl](https://github.com/JuliaNLSolvers/LsqFit.jl) for this task.

**Note:** this problem is usually modeled as a numerical optimizaiton problem.
"""

# ╔═╡ 4a7d2c25-04b1-4ccb-85cd-7276902a9679
md"""
### Generate data

We will first set up our data and plot it...
"""

# ╔═╡ 7203ba1b-ce21-4c53-9dce-358df13df21e
begin
	x = 0:0.05:10
	y = 1 * exp.(-x * 2) + 2 * sin.(0.8 * π * x) + 0.15 * randn(length(x));
	scatter(x, y, legend=false)
end

# ╔═╡ 775e6fbe-8c10-4f94-b53e-618e4dd551fc
md"""
### Modeling data

Then, we set up the model with `model(x,p)`. The vector `p` is what to be estimated given a set of values `x`.
"""

# ╔═╡ 5d75ebce-1181-4490-ae10-8bcc08456fdf
begin
	@. model(v, p) = p[1] * exp(-v * p[2]) + p[3] * sin(0.8 * π *v)
	p0 = [0.5, 0.5, 0.5]
	myfit = curve_fit(model, x, y, p0)
end

# ╔═╡ a94163fe-e1ac-483a-bc2d-5d7795cfe188
md"""
⚠ A note about `curve_fit`: this function can take multiple other inputs, for instance the Jacobian of what you are trying to fit. We don't dive into these details here, but be sure to check out the LsqFit.jl package to see what other things you can can pass to create a better fit.

Also note that julia has multiple packages that allow you to create Jacobians so you don't have to write them yourself. Two such packages are [FiniteDifferences.jl](https://github.com/JuliaDiff/FiniteDifferences.jl) or [ForwardDiff.jl](https://github.com/JuliaDiff/ForwardDiff.jl).

↪ Back to our example. Let's define the paramenter variable and then calculate the predictions `ŷ`...
"""

# ╔═╡ 6a965e6d-4724-47c8-a3bb-bfec7d6706c9
begin
	p = myfit.param
	ŷ = p[1] * exp.(-x * p[2]) + p[3] * sin.(0.8 * π * x)
end

# ╔═╡ c7d07a47-3f3f-482e-9c06-78877b91e380
md"""
We are ready now to plot the curve we have generated...
"""

# ╔═╡ 6ca3290d-5980-4749-b309-fd84d73b7bb5
begin
	scatter(x, y, legend=false, xlabel="X", ylabel="Y")
	plot!(x, ŷ)
end

# ╔═╡ Cell order:
# ╟─d16725a0-6cd3-11ec-0998-fd4af26f10dd
# ╟─91471950-703f-4927-8bce-f4478f4a53ba
# ╟─a53d44c5-75c4-4508-aa19-33e838c9e0bb
# ╟─e9458f99-4ec0-4167-b771-3090dfec6320
# ╟─784c5b0e-59dc-4b2d-bf40-112261ac8e85
# ╠═65e22edc-ef5c-412f-82f4-1c84da209d6d
# ╟─db2c3459-2127-4b46-8e73-546850767dc0
# ╠═3bb8c382-f97d-48dc-8cbe-99bf77de0784
# ╟─75059ff4-6b9d-4e01-9aed-33ea8b0e9a71
# ╠═e83331a1-8655-4de0-a36f-b74b5e43674b
# ╟─d81e448b-b4e4-4d6f-8ca7-42c41569977d
# ╠═a047abfb-178c-40ac-8f8c-c3a8a73c1849
# ╠═208af138-5131-4034-885b-c41f6ccf36f5
# ╠═39884fc4-c41c-46e5-a92a-44d766243715
# ╟─13719feb-60ae-433b-bf1e-df6ac045215d
# ╠═10c072d4-b793-458c-b947-d5dc3aae5e25
# ╠═68a0574f-5b17-48ae-b8c4-a03c7e385561
# ╠═ecb1cb29-2b57-4197-9327-e762fc876281
# ╟─056b28f3-8da4-4a94-a33c-72b30c8aa0aa
# ╟─9a5dcad1-9365-423c-b27a-a6c206850848
# ╟─a89ec73a-5e35-49ad-8829-dc2ecc7e8aac
# ╠═aa78c706-8860-46f1-9982-ea10bdc4fccb
# ╟─618ab9ee-ee9c-4dbb-b48f-ebeeb15549c6
# ╠═b530e842-bfa4-4c41-9cc9-4aa629e12f99
# ╠═041e87c7-deee-46c1-85be-ea47df16217c
# ╟─19a44596-2367-4964-b21e-7d84ba147915
# ╠═ab8cf8d2-9ac6-4361-b168-243f781e7dfa
# ╟─215483a1-6059-437d-a4b6-ca3cd897f13b
# ╠═b075d090-91aa-4d1f-bb9f-a93dd65413e7
# ╠═1b8f7b0d-a843-4af7-90b7-4a64d74edab0
# ╟─39456bea-24dc-482b-9d8b-33a9f2ed715a
# ╠═f195db71-f2a4-4f5c-bfa4-2d9ec1f23a32
# ╟─2da58c4e-8db4-4d80-aa29-f3ec10952f73
# ╠═93179d9a-5bf4-488b-b710-28a5f56b2e26
# ╟─560a134f-1175-4d56-83af-1ad6bbe1a99f
# ╠═0c4a00f6-69f2-4fc3-a7f8-ca6e38404ac9
# ╠═a02a9fdf-46bc-4ea1-bb29-b1731a66c057
# ╠═63b596be-9392-4efb-8d31-7f1d6e9c7254
# ╠═feced955-2e69-44ae-bc2a-82d4f2136b38
# ╟─831484f5-6cdc-4641-8adf-117a33c518c2
# ╠═b94019d8-a7d8-4781-819b-28eeac823812
# ╟─4735fc87-68cb-4617-aa7c-6139ba7b7c0a
# ╠═ec7ae9e6-b191-4578-8a47-9d2245280316
# ╟─216dc17e-54c1-4520-890d-8663165b10dd
# ╠═dddfb8fb-7047-4417-8dd2-569a93277f0c
# ╟─5116c117-75e4-4989-b06f-2883bba48330
# ╠═3a86069b-3f24-4eec-a7ae-f77189a4fd77
# ╠═453385dd-8bda-42b8-9b4a-6a4963b21346
# ╠═bd31693d-dee9-4534-b4d4-3904997e8020
# ╟─0fb142cd-ac45-42e8-b5d1-24c6142392bb
# ╠═3caa5331-2507-4299-b2e4-50e8a5cbbe46
# ╟─22c201af-9437-4dde-8653-508ebf514008
# ╠═16367b7a-65ea-4fa0-abad-a1cc25c60ee6
# ╠═1d7ef9db-dd15-4ed4-841a-b4baf02302a1
# ╠═f42bdc3a-943b-4157-853d-860af52e059c
# ╟─310f2282-d6dd-405d-8399-e75fdc75c27c
# ╠═cc911a53-9139-4c90-b02b-15f30e6772f0
# ╟─79bbe213-c15a-4ecb-aef7-107846234f1d
# ╠═11b1f5d1-7ae2-43dd-bd35-5b0ba6fde6f4
# ╠═e3e9d570-af57-4065-84ec-b61400fa449d
# ╟─dfc76a4c-4e2b-4588-92e1-2209507fd670
# ╟─31308555-dfc9-4e28-9108-8a83d65b7770
# ╟─4a7d2c25-04b1-4ccb-85cd-7276902a9679
# ╠═7203ba1b-ce21-4c53-9dce-358df13df21e
# ╟─775e6fbe-8c10-4f94-b53e-618e4dd551fc
# ╠═5d75ebce-1181-4490-ae10-8bcc08456fdf
# ╟─a94163fe-e1ac-483a-bc2d-5d7795cfe188
# ╠═6a965e6d-4724-47c8-a3bb-bfec7d6706c9
# ╟─c7d07a47-3f3f-482e-9c06-78877b91e380
# ╠═6ca3290d-5980-4749-b309-fd84d73b7bb5


# Classical Multidimensional Scaling

## convert Gram matrix to Distance matrix

"""
    gram2dmat!(D, G)

Convert a Gram matrix `G` to a distance matrix, and write the results to `D`.
"""
function gram2dmat!(D::AbstractMatrix{DT}, G::AbstractMatrix) where {DT<:Real}
    # argument checking
    m = size(G, 1)
    n = size(G, 2)
    m == n || error("D should be a square matrix.")
    size(D) == (m, n) ||
        throw(DimensionMismatch("Sizes of D and G do not match."))

    # implementation
    for j = 1:n
        for i = 1:j-1
            @inbounds D[i,j] = D[j,i]
        end
        D[j,j] = zero(DT)
        for i = j+1:n
            @inbounds D[i,j] = sqrt(G[i,i] + G[j,j] - 2 * G[i,j])
        end
    end
    return D
end

"""
    gram2dmat(G)

Convert a Gram matrix `G` to a distance matrix.
"""
gram2dmat(G::AbstractMatrix{T}) where {T<:Real} = gram2dmat!(similar(G, T), G)

## convert Distance matrix to Gram matrix

"""
    dmat2gram!(G, D)

Convert a distance matrix `D` to a Gram matrix, and write the results to `G`.
"""
function dmat2gram!(G::AbstractMatrix{GT}, D::AbstractMatrix) where GT
    # argument checking
    n = LinearAlgebra.checksquare(D)
    size(G) == (n, n) ||
        throw(DimensionMismatch("Sizes of G and D do not match."))

    # implementation
    u = zeros(GT, n)
    s = 0.0
    for j = 1:n
        s += (u[j] = sum(abs2, view(D,:,j)) / n)
    end
    s /= n

    for j = 1:n
        for i = 1:j-1
            @inbounds G[i,j] = G[j,i]
        end
        for i = j:n
            @inbounds G[i,j] = (u[i] + u[j] - abs2(D[i,j]) - s) / 2
        end
    end
    return G
end

momenttype(T) = typeof((zero(T) * zero(T) + zero(T) * zero(T))/ 2)

"""
    dmat2gram(D)

Convert a distance matrix `D` to a Gram matrix.
"""
dmat2gram(D::AbstractMatrix{T}) where {T<:Real} = dmat2gram!(similar(D, momenttype(T)), D)

## Classical MDS

"""
*Classical Multidimensional Scaling* (MDS), also known as Principal Coordinates Analysis (PCoA),
is a specific technique in this family that accomplishes the embedding in two steps:

1. Convert the distance matrix to a Gram matrix. This conversion is based on
the following relations between a distance matrix ``D`` and a Gram matrix ``G``:

```math
\\mathrm{sqr}(\\mathbf{D}) = \\mathbf{g} \\mathbf{1}^T + \\mathbf{1} \\mathbf{g}^T - 2 \\mathbf{G}
```

Here, ``\\mathrm{sqr}(\\mathbf{D})`` indicates the element-wise square of ``\\mathbf{D}``,
and ``\\mathbf{g}`` is the diagonal elements of ``\\mathbf{G}``. This relation is
itself based on the following decomposition of squared Euclidean distance:

```math
\\| \\mathbf{x} - \\mathbf{y} \\|^2 = \\| \\mathbf{x} \\|^2 + \\| \\mathbf{y} \\|^2 - 2 \\mathbf{x}^T \\mathbf{y}
```

2. Perform eigenvalue decomposition of the Gram matrix to derive the coordinates.

*Note:*  The Gramian derived from ``D`` may have non-positive or degenerate
eigenvalues.  The subspace of non-positive eigenvalues is projected out
of the MDS solution so that the strain function is minimized in a
least-squares sense.  If the smallest remaining eigenvalue that is used
for the MDS is degenerate, then the solution is not unique, as any
linear combination of degenerate eigenvectors will also yield a MDS
solution with the same strain value.
"""
struct MDS{T<:Real} <: NonlinearDimensionalityReduction
    d::Real                  # original dimension
    X::AbstractMatrix{T}     # fitted data, X (d x n)
    λ::AbstractVector{T}     # eigenvalues in feature space, (k x 1)
    U::AbstractMatrix{T}     # eigenvectors in feature space, U (n x k)
end

## properties
"""
    size(M)

Returns tuple where the first value is the MDS model `M` input dimension,
*i.e* the dimension of the observation space, and the second value is the output
dimension, *i.e* the dimension of the embedding.
"""
size(M::MDS) = (M.d, size(M.U,2))

"""
    projection(M)

Get the MDS model `M` eigenvectors matrix (of size ``(n, p)``) of the embedding space.
The eigenvectors are arranged in descending order of the corresponding eigenvalues.
"""
projection(M::MDS) = M.U

"""
    eigvecs(M::MDS)

Get the MDS model `M` eigenvectors matrix.
"""
eigvecs(M::MDS) = projection(M)

"""
    eigvals(M::MDS)

Get the eigenvalues of the MDS model `M`.
"""
eigvals(M::MDS) = M.λ

"""
    loadings(M::MDS)

Get the loading of the MDS model `M`.
"""
loadings(M::MDS) = sqrt.(M.λ)' .* M.U

## use

"""
    predict(M, x::AbstractVector)

Calculate the out-of-sample transformation of the observation `x` for the MDS model `M`.
Here, `x` is a vector of length `d`.
"""
function predict(M::MDS, x::AbstractVector{T}; distances=false) where {T<:Real}
    d = if isnan(M.d) # model has only distance matrix
        @assert distances "Cannot transform points if model was fitted with a distance matrix. Use point distances."
        size(x, 1) != size(M.X, 1) && throw(
            DimensionMismatch("Point distances should be calculated to all original points"))
        x
    else
        if distances
            size(x, 1) != size(M.X, 2) && throw(
                DimensionMismatch("Point distances should be calculated to all original points."))
            x
        else
            size(x, 1) != size(M.X, 1) && throw(
                DimensionMismatch("Points and original data must have same dimensionality."))
            pairwise((x,y)->norm(x-y), M.X, x)
        end
    end

    # get distance matrix
    D = isnan(M.d) ? M.X : pairwise((x,y)->norm(x-y), eachcol(M.X), symmetric=true)
    d = d.^2

    # b = 0.5*(ones(n,n)*d./n - d + D*ones(n,1)./n - ones(n,n)*D*ones(n,1)./n^2)
    mD = mean(D.^2, dims=2)
    b = (d  .- mean(d, dims=1) .- mD .+ mean(mD)) ./ -2

    # sqrt(λ)⁻¹U'b
    λ = vcat(M.λ, zeros(T, size(M)[2] - length(M.λ)))
    return M.U' * b ./ sqrt.(λ)
end

"""
    predict(M)

Returns a coordinate matrix of size ``(p, n)`` for the MDS model `M`, where each column
is the coordinates for an observation in the embedding space.
"""
function predict(M::MDS{T}) where {T<:Real}
    d, p = size(M)
    # if there are non-positive missing eigval then pad with zeros
    λ = vcat(M.λ, zeros(T, p - length(M.λ)))
    return diagm(0=>sqrt.(λ)) * M.U'
end

## show

function show(io::IO, M::MDS)
    d, p = size(M)
    print(io, "Classical MDS(indim = $d, outdim = $p)")
end

## interface functions

"""
    fit(MDS, X; kwargs...)

Compute an embedding of `X` points by classical multidimensional scaling (MDS).
There are two calling options, specified via the required keyword argument `distances`:

    mds = fit(MDS, X; distances=false, maxoutdim=size(X,1)-1)

where `X` is the data matrix. Distances between pairs of columns of `X` are computed using the Euclidean norm.
This is equivalent to performing PCA on `X`.

    mds = fit(MDS, D; distances=true, maxoutdim=size(D,1)-1)

where `D` is a symmetric matrix `D` of distances between points.
"""
function fit(::Type{MDS}, X::AbstractMatrix{T};
             maxoutdim::Int = size(X,1)-1,
             distances::Bool) where T<:Real

    # get distance matrix and space dimension
    D, d = if !distances
        pairwise((x,y)->norm(x-y), eachcol(X), symmetric=true), size(X,1)
    else
        X, NaN
    end
    G = dmat2gram(D)

    n = size(D, 1)
    m = min(maxoutdim, n) #Actual number of eigenpairs wanted

    E = eigen!(Hermitian(G))

    #Sometimes dmat2gram produces a negative definite matrix, and the sign just
    #needs to be flipped. The heuristic to check for this robustly is to check
    #if there is a negative eigenvalue of magnitude larger than the largest
    #positive eigenvalue, and flip the sign of eigenvalues if necessary.
    mineig, maxeig = extrema(E.values)
    if mineig < 0 && abs(mineig) > abs(maxeig)
        #do flip
        ord = sortperm(E.values)
        λ = -E.values[ord[1:m]]
    else
        ord = sortperm(E.values; rev=true)
        λ = E.values[ord[1:m]]
    end

    for i in 1:m
        if λ[i] <= 0
            #Keeping all remaining eigenpairs would not change solution (if 0)
            #or make the answer _worse_ (if <0).
            #The least squares solution would want to throw all these away.
            @warn("Gramian has only $(i-1) positive eigenvalue(s)")
            m = i-1
            ord = ord[1:m]
            λ = λ[1:m]
            break
        end
    end

    #Check if the last considered eigenvalue is degenerate
    if m>0
        nevalsmore = sum(abs.(E.values[ord[m+1:end]] .- λ[m]) .< n*eps())
        nevals = sum(abs.(E.values .- λ[m]) .< n*eps())
        if nevalsmore > 1
            @warn("The last eigenpair is degenerate with $(nevals-1) others; $nevalsmore were ignored. Answer is not unique")
        end
    end
    U = E.vectors[:, ord[1:m]]

    #Add trailing zero coordinates if dimension of embedding space (p) exceeds
    #number of eigenpairs used (m)
    if m < maxoutdim
        U = [U zeros(T, n, maxoutdim-m)]
    end

    return MDS(d, X, λ, U)
end

"""
    stress(M)

Get the stress of the MDS mode `M`.
"""
function stress(M::MDS)
    # calculate distances if original data was stored
    DX = isnan(M.d) ? M.X : pairwise((x,y)->norm(x-y), eachcol(M.X), symmetric=true)
    DY = pairwise((x,y)->norm(x-y), eachcol(predict(M)), symmetric=true)
    n = size(DX,1)
    return sqrt(2*sum((DX - DY).^2)/sum(DX.^2));
end