#Generate 2D data from 4 spherical (σ=0.25) gaussian clusters centered at [-1,1], [1,-1], [-1,1] and [1,1]
"""
    generate_4_clusters(npts_per_cluster; σ=0.25)
    generate_4_clusters(::DataType, npts_per_cluster; σ=0.25)

Generate 2D data from 4 spherical gaussian clusters, with standard deviation `σ`, centered at [-1,1], [1,-1], [-1,1] and [1,1]. `DataType` can be `Float32` or `Float64`, default is `Float64`.
"""
function generate_4_clusters(T::DataType, npts_per_cluster; σ = 0.25)
    c = convert(Matrix{T},[1 1 -1 -1; 1 -1 1 -1])
    data = Matrix{T}(undef,2,4*npts_per_cluster)
    for i ∈ axes(c,2)
        rpts = randn(T,(2,npts_per_cluster))
        sindx = (i-1)*npts_per_cluster+1
        eindx = i*npts_per_cluster
        data[:,sindx:eindx] = σ*rpts .+ c[:,i]
    end
    return data
end

generate_4_clusters(npts_per_cluster; σ = 0.25) = generate_4_clusters(Float64, npts_per_cluster; σ)

#Generate 2D data arranged in a circle (default radius, r =0.5) around centres at [-1,1], [1,-1], [-1,1] and [1,1]
"""
    generate_4_circle_clusters(npts_per_cluster = 8; r = 0.5)
    generate_4_circle_clusters(::DataType, npts_per_cluster = 8; r = 0.5)

Generate 2D data arranged in a circle (default radius, r =0.5) around centres at [-1,1], [1,-1], [-1,1] and [1,1]. `DataType` can be `Float32` or `Float64`, default is `Float64`.
"""
function generate_4_circle_clusters(T::DataType, npts_per_cluster::Int = 8; r = 0.5)
    c = convert(Matrix{T},[1 1 -1 -1; 1 -1 1 -1])
    data = Matrix{T}(undef,2,4*npts_per_cluster)
    for i ∈ axes(c,2)
        sindx = (i-1)*npts_per_cluster+1
        eindx = i*npts_per_cluster
        pts = generate_circle_cluster(T, npts_per_cluster)
        data[:,sindx:eindx] = r*pts .+ c[:,i]
    end
    return data
end

generate_4_circle_clusters(npts_per_cluster::Int = 8; r = 0.5) = generate_4_circle_clusters(Float64, npts_per_cluster; r)

#Generate 2D data arranged in a circle of radius one around [0,0]
"""
    generate_circle_cluster(npts = 8)
    generate_circle_cluster(::DataType, npts = 8)

Generate 2D data arranged in a circle of radius one around [0,0]. `DataType` can be `Float32` or `Float64`, default is `Float64`.
"""
function generate_circle_cluster(T::DataType, npts::Int = 8)
    Δθ = 2π/npts
    θ = 0:Δθ:2π-Δθ
    x = Matrix{T}(undef,2,npts)
    for i ∈ eachindex(θ)
        x[1,i] = cos(θ[i])
        x[2,i] = sin(θ[i])
    end
    return x
end

generate_circle_cluster(npts::Int = 8) = generate_circle_cluster(Float64, npts)


A = [ 1. 2 3; 4 5 6; 7 8 9 ]

B = [12 -51 4; 6 167 -68; -4 24 -41]

function check_DV(A, D, V)
    I = eye(size(A)[1])
    for i = 1:size(A)[1]
        check = (A - D[i] * I) * V[:,i]
        println("For the ", i, "th eigen value and vector, (A - lambda I)^k * v = ", check, "\n\twhich has a L2 norm = ", norm(check))
        assert(norm(check) <= 1e-14)
    end
end

D_A, V_A = eig(A)

check_DV(A, D_A, V_A)

D_B, V_B = eig(B)

check_DV(B, D_B, V_B)

"Power iteration method on A, to find eigenpair with largest eigen value."
function poweriteration(A, maxiter=20, userandomstart=true)
    n = size(A)[1]
    if userandomstart
        X = randn((n, 1))
    else
        X = ones((n, 1))
    end
    for i = 1:maxiter
        next_X = A * X
        X = next_X / norm(next_X)
    end
    # lambda = (X^* ⋅ A ⋅ X) / (X^* ⋅ X)
    lambda = (transpose(conj(X)) * (A * X)) / (transpose(conj(X)) * X)
    lambda, X
end

poweriteration(A)

D_A, V_A = eig(A)

(D_A[1], -1 * V_A[:,1])

poweriteration

println(poweriteration(A, 1))
println(poweriteration(A, 2))
println(poweriteration(A, 5))
println(poweriteration(A, 10))

"Inner product of two vectors v, w. Can be vertical (n,1) or horizontal (1,n) vectors."
function inner(v, w)
    assert(size(v) == size(w))
    nm = size(v)
    if length(nm) == 1
        return transpose(conj(v)) * w
    elseif nm[1] == 1
        return conj(v) * transpose(w)
    end
end

inner([1 1 1], [2 3 4])

inner([1; 1; 1], [2; 3; 4])

"projection(a, e): projection operator of vector a onto base vector e. Uses inner(e, a) and inner(e, e)."
function projection(a, e)
    return (inner(e, a) / inner(e, e)) * e
end

"gramschmidt(A): Gram-Schmidt orthogonalization operator, returns us, es (unormalized matrix, base matrix)."
function gramschmidt(A, verbose=false)
    assert(size(A)[1] == size(A)[2])
    n = size(A)[1]
    if verbose
        println("n = ", n)
    end
    us = zeros((n, n))
    es = zeros((n, n))
    for i = 1:n
        if verbose
            println("i = ", i)
        end
        us[:,i] = A[:,i]
        for j = 1:i-1
            if verbose
                println("\tj = ", j)
                println("\tus[:,i] = ", us[:,i])
            end
            us[:,i] -= projection(A[:,i], us[:,j])
        end
        if verbose
            println("us[:,i] = ", us[:,i])
        end
        es[:,i] = us[:,i] / norm(us[:,i])
        if verbose
            println("es[:,i] = ", es[:,i])
        end
    end
    return us, es
end

A

us, es = gramschmidt(A)

us

es

inner(es[:,1], A[:,1]) * es[:,1]

inner(es[:,1], A[:,2]) * es[:,1] + inner(es[:,2], A[:,2]) * es[:,2]

rank(A)

inner(es[:,1], A[:,3]) * es[:,1] + inner(es[:,2], A[:,3]) * es[:,2] + inner(es[:,3], A[:,3]) * es[:,3]

[ norm(es[:,i]) for i = 1:size(es)[1] ]

(es' * es)[1:2,1:2]

us, es = gramschmidt(B)

[ norm(es[:,i]) for i = 1:size(es)[1] ]

es * es'

V = [3 2; 1 2]

us, es = gramschmidt(V)

es' * es

"QR decomposition, returns (Q, R): Q is orthogonal and R is upper-triangular, such that A = Q R."
function QR(A)
    assert(size(A)[1] == size(A)[2])
    n = size(A)[1]
    us, es = gramschmidt(A)
    Q = copy(es)
    R = zeros((n, n))
    for i = 1:n
        for j = i:n
            R[i,j] = inner(es[:,i], A[:,j])
        end
    end
    return Q, R
end

Q, R = QR(A)

A

Q

R

Q * R

Q2, R2 = QR(B)

B

Q2

R2

Q2 * R2

B == Q2 * R2

assert(norm(B - (Q2 * R2)) <= 1e-13)

"Apply the QR method for maxiter steps. Should return a triangular matrix similar to A (same eigenvalues)."
function QR_method(A, maxiter=50)
    Ak = A
    for k = 1:maxiter
        Qk, Rk = QR(Ak)
        Ak = Rk * Qk
    end
    return Ak
end

Ak = QR_method(A)

"Truncate to zero values under the diagonal (have to be smaller than tolerance)"
function truncate_to_zero_below_diag(A, tolerance=1e-12)
    assert(size(A)[1] == size(A)[2])
    n = size(A)[1]
    for j = 1:n
        for i = j+1:n
            assert(norm(A[i,j]) <= tolerance)
            A[i,j] = 0
        end
    end
    return A
end

truncate_to_zero_below_diag(Ak)

Ak = QR_method(B)

truncate_to_zero_below_diag(Ak)

"Extract the diagonal from the QR method."
function QR_eigenvalues(A, maxiter=50)
    return diag(QR_method(A, maxiter))
end

QR_eigenvalues(A)

eigvals(A)

QR_eigenvalues(B)

eigvals(B)

"Inverse iteration method to find the eigen vector corresponding to a given eigen value."
function inverseIteration(A, val, maxiter=20, userandomstart=true)
    mu_I = val * eye(A)
    inv_A_minus_muI = inv(A - mu_I)
    n = size(A)[1]
    if userandomstart
        X = randn((n, 1))
    else
        X = ones((n, 1))
    end
    for i = 1:maxiter
        next_X = inv_A_minus_muI * X
        X = next_X / norm(next_X)
    end
    X
end

"Approximation of D, V: D contains the eigen values (vector) and V the eigen vectors (column based)."
function QR_eigen(A, maxiter=20, userandomstart=true)
    n = size(A)[1]
    D = QR_eigenvalues(A, maxiter)
    V = zeros((n,n))
    for i = 1:n
        V[:,i] = inverseIteration(A, D[i], maxiter, userandomstart)
    end
    return D, V
end

D2, V2 = QR_eigen(B)

D2s, V2s = eig(B)


using NumericalMethodsforEngineers

a = [1. 2. -2.; 2. 5. -4.; -2. -4. 5.]

println("\n|a[1, 1]| = $(det(a[1, 1]))\n")
println("|a[1:2, 1:2]| = $(det(a[1:2, 1:2])))\n")
println("|a| = $(det(a))")
println("\nchol(a):")
chol(a) |> display

a = [16. 4. 8.; 4. 5. -4.; 8. -4. 22.]

println("\n|a[1, 1]| = $(det(a[1, 1]))\n")
println("|a[1:2, 1:2]| = $(det(a[1:2, 1:2])))\n")
println("|a| = $(det(a))")

println("\nchol(a) - upper:")
upper = chol(a)
upper |> display

println("\nchol(a) - lower:")
lower = chol(a)'
lower |> display

println("\nlower * upper:")
lower * upper |> display

@test a == lower * upper


module RandomTestMatrices

export randsym, randantisym, randposdef, randstable, randnormal, randorth, randunitary

#=

Random test matrices. These are for testing and debugging purposes, for random matrix ensembles with known limiting distributions, see the package RandomMatrices.jl .

=#

"""
randsym(d)

	Gaussian random symmetric matrix of dimension ``d``.
"""
function randsym(d)
    y = randn(d, d)
	triu(y) + triu(y,1)'
end

"""
randantisym(d)

    Anti symmetric matrix
"""
function randantisym(d)
	y = randn(d, d)
	triu(y,1) - triu(y,1)'
end

"""
randposdef(d, n = d)

	Random positive definite matrix of dimension ``d`` from the Laguerre ensemble ``n``.
"""
function randposdef(d,n = d)
	x = randn(d, n)
	x'* x/n
end

"""
randstable(d)

	Random stable matrix (matrix with eigenvalues with negative real part) with dimension ``d``.
"""
function randstable(d)
	# positive definite matrix
	x = randn(d, d)
	a = x'*x/d

	# antisymmetric matrix
	b = randantisym(d)

	# return stable matrix
	b - a
end

"""
randunitary(d)

	Random unitary matrix of dimension ``d``.
"""
randunitary(d) = expm(im* randposdef(d))

"""
randorth(d)

	Orthogonal matrix drawn according to the Haar measure on the group of orthogonal matrices.
"""
randorth(d) = qr(randn(d,d))[1]

function tridiag(di, u, l)
	assert(length(di) -1 == length(u) == length(l))
	M = diagm(di)
	for i in 1:length(u)
	  M[i,i+1] = l[i]
  	  M[i+1,i] = u[i]
	end
	M

end

"""
randnormal(d)

	Random normal matrix of dimension ``d``.
"""
function randnormal(d)
	Q = randorth(d)
	B = schur(randn(d,d))[1] #same eigenvalues as a randn(d,d)
	m = d
	alpha = zeros(d)
	beta = zeros(d-1)
	while m > 1
	   s = abs(B[m-1,m-1]) + abs(B[m,m])
	   if s + abs(B[m,m-1]) > s # if significant offdiagonal value: evaluate submatrix m-1:m;
		spur = B[m,m] + B[m-1,m-1]
		dis2 = abs2(B[m,m]- B[m-1,m-1]) + 4.0*B[m,m-1]*B[m-1,m] # = spur^2 - 4det
		alpha[m] = alpha[m-1] = 0.5*spur

		beta[m-1] = 0.5*sqrt(-dis2)
		m -= 2
	   else
	        alpha[m] = B[m,m]
	        beta[m-1] = 0
	        m -= 1
	   end

	end
	if (m == 1)
		alpha[1] = B[1,1]
	end

	Q' * tridiag(alpha, beta, -beta) * Q

end

end # module


# # Factorizations and other fun
# Based on work by Andreas Noack
#
# ## Outline
#  - Factorizations
#  - Special matrix structures
#  - Generic linear algebra

#-

# Before we get started, let's set up a linear system and use `LinearAlgebra` to bring in the factorizations and special matrix structures.

using LinearAlgebra
A = rand(3, 3)
x = fill(1, (3,))
b = A * x

# ## Factorizations
#
# #### LU factorizations
# In Julia we can perform an LU factorization
# ```julia
# PA = LU
# ```
# where `P` is a permutation matrix, `L` is lower triangular unit diagonal and `U` is upper triangular, using `lufact`.
#
# Julia allows computing the LU factorization and defines a composite factorization type for storing it.

Alu = lu(A)

#-

typeof(Alu)

# The different parts of the factorization can be extracted by accessing their special properties

Alu.P

#-

Alu.L

#-

Alu.U

# Julia can dispatch methods on factorization objects.
#
# For example, we can solve the linear system using either the original matrix or the factorization object.

A\b

#-

Alu\b

# Similarly, we can calculate the determinant of `A` using either `A` or the factorization object

det(A) ≈ det(Alu)

# #### QR factorizations
#
# In Julia we can perform a QR factorization
# ```
# A=QR
# ```
#
# where `Q` is unitary/orthogonal and `R` is upper triangular, using `qrfact`.

Aqr = qr(A)

# Similarly to the LU factorization, the matrices `Q` and `R` can be extracted from the QR factorization object via

Aqr.Q

#-

Aqr.R

# #### Eigendecompositions

#-

# The results from eigendecompositions, singular value decompositions, Hessenberg factorizations, and Schur decompositions are all stored in `Factorization` types.
#
# The eigendecomposition can be computed

Asym = A + A'
AsymEig = eigen(Asym)

# The values and the vectors can be extracted from the Eigen type by special indexing

AsymEig.values

#-

AsymEig.vectors

# Once again, when the factorization is stored in a type, we can dispatch on it and write specialized methods that exploit the properties of the factorization, e.g. that $A^{-1}=(V\Lambda V^{-1})^{-1}=V\Lambda^{-1}V^{-1}$.

inv(AsymEig)*Asym

# ## Special matrix structures
# Matrix structure is very important in linear algebra. To see *how* important it is, let's work with a larger linear system

n = 1000
A = randn(n,n);

# Julia can often infer special matrix structure

Asym = A + A'
issymmetric(Asym)

# but sometimes floating point error might get in the way.

Asym_noisy = copy(Asym)
Asym_noisy[1,2] += 5eps()

#-

issymmetric(Asym_noisy)

# Luckily we can declare structure explicitly with, for example, `Diagonal`, `Triangular`, `Symmetric`, `Hermitian`, `Tridiagonal` and `SymTridiagonal`.

Asym_explicit = Symmetric(Asym_noisy);

# Let's compare how long it takes Julia to compute the eigenvalues of `Asym`, `Asym_noisy`, and `Asym_explicit`

@time eigvals(Asym);

#-

@time eigvals(Asym_noisy);

#-

@time eigvals(Asym_explicit);

# In this example, using `Symmetric()` on `Asym_noisy` made our calculations about `5x` more efficient :)

#-

# #### A big problem
# Using the `Tridiagonal` and `SymTridiagonal` types to store tridiagonal matrices makes it possible to work with potentially very large tridiagonal problems. The following problem would not be possible to solve on a laptop if the matrix had to be stored as a (dense) `Matrix` type.

n = 1_000_000;
A = SymTridiagonal(randn(n), randn(n-1));
@time eigmax(A)

# ## Generic linear algebra
# The usual way of adding support for numerical linear algebra is by wrapping BLAS and LAPACK subroutines. For matrices with elements of `Float32`, `Float64`, `Complex{Float32}` or `Complex{Float64}` this is also what Julia does.
#
# However, Julia also supports generic linear algebra, allowing you to, for example, work with matrices and vectors of rational numbers.

#-

# #### Rational numbers
# Julia has rational numbers built in. To construct a rational number, use double forward slashes:

1//2

# #### Example: Rational linear system of equations
# The following example shows how linear system of equations with rational elements can be solved without promoting to floating point element types. Overflow can easily become a problem when working with rational numbers so we use `BigInt`s.

Arational = Matrix{Rational{BigInt}}(rand(1:10, 3, 3))/10

#-

x = fill(1, 3)
b = Arational*x

#-

Arational\b

#-

lu(Arational)

# ### Exercises
#
# #### 11.1
# What are the eigenvalues of matrix A?
#
# ```
# A =
# [
#  140   97   74  168  131
#   97  106   89  131   36
#   74   89  152  144   71
#  168  131  144   54  142
#  131   36   71  142   36
# ]
# ```
# and assign it a variable `A_eigv`

using LinearAlgebra

#-

@assert A_eigv ==  [-128.49322764802145, -55.887784553056875, 42.7521672793189, 87.16111477514521, 542.4677301466143]

# #### 11.2
# Create a `Diagonal` matrix from the eigenvalues of `A`.

@assert A_diag ==  [-128.493    0.0      0.0      0.0       0.0;
    0.0    -55.8878   0.0      0.0       0.0;
    0.0      0.0     42.7522   0.0       0.0;
    0.0      0.0      0.0     87.1611    0.0;
    0.0 0.0      0.0      0.0     542.468]
## #### 11.3
## Create a `LowerTriangular` matrix from `A` and store it in `A_lowertri`
@assert A_lowertri ==  [140    0    0    0   0;
  97  106    0    0   0;
  74   89  152    0   0;
 168  131  144   54   0;
 131   36   71  142  36]


# Mld.jl

###### mld #####
"""
    mld(v)

Compute the Mean log deviation of a vector `v`.

# Examples
```julia
julia> using Inequality
julia> mld([8, 5, 1, 3, 5, 6, 7, 6, 3])
0.1397460530936332
```
"""
mld(v::AbstractVector{<:Real})::Float64 = log(Statistics.mean(v[v .!= 0])) - Statistics.mean(log.(v[v .!= 0]))

###### weighted mld #####
"""
    mld(v, w)

Compute the weighted Mean log deviation of a vector `v` using weights given by a weight vector `w`.

Weights must not be negative, missing or NaN. The weights and data vectors must have the same length.

# Examples
```julia
julia> using Inequality
julia> mld([8, 5, 1, 3, 5, 6, 7, 6, 3], collect(0.1:0.1:0.9))
0.10375545537468206
```
"""
function mld(v::AbstractVector{<:Real}, w::AbstractVector{<:Real})::Float64
    checks_weights(v, w)
    w = w[v .!= 0]
    v = v[v .!= 0]

    return -sum(w .* log.(v/mean(v, weights(w))) ) / sum(w)
end

function mld(v::AbstractVector{<:Real}, w::AbstractWeights)::Float64
    checks_weights(v, w)
    w = w[v .!= 0]
    v = v[v .!= 0]

    return -sum(w .* log.(v/mean(v, weights(w))))/ sum(w)
end

"""
    wmld(v, w)

Compute the Mean log deviationof `v` with weights `w`. See also [`mld`](@mld)
"""
wmld(v::AbstractVector{<:Real}, w::AbstractVector{<:Real}) = mld(v, w)

wmld(v::AbstractVector{<:Real}, w::AbstractWeights) = mld(v, w)


struct ODEProblem{F,T<:Tuple{Number,Number},U<:AbstractVector,P<:AbstractVector}
    f::F
    tspan::T
    u0::U
    θ::P
end

abstract type ODESolver end

struct Euler{T} <: ODESolver
    dt::T
end

function (solver::Euler)(prob::ODEProblem, u, t)
    f, θ, dt  = prob.f, prob.θ, solver.dt
    (u + dt*f(u,θ), t+dt)
end

function solve(prob::ODEProblem, solver::ODESolver)
    t = prob.tspan[1]; u = prob.u0
    us = [u]; ts = [t]
    while t < prob.tspan[2]
        (u,t) = solver(prob, u, t)
        push!(us,u)
        push!(ts,t)
    end
    ts, reduce(hcat,us)
end

# Define & Solve ODE

function lotkavolterra(x,θ)
    α, β, γ, δ = θ
    x₁, x₂ = x

    dx₁ = α*x₁ - β*x₁*x₂
    dx₂ = δ*x₁*x₂ - γ*x₂

    [dx₁, dx₂]
end

θ = [0.1,0.2,0.3,0.2]
u0 = [1.0,1.0]
tspan = (0.,100.)
prob = ODEProblem(lotkavolterra,tspan,u0,θ)


"""
```julia
lorenz(u0=[0.0, 10.0, 0.0]; σ = 10.0, ρ = 28.0, β = 8/3) -> ds
```
```math
\\begin{aligned}
\\dot{X} &= \\sigma(Y-X) \\\\
\\dot{Y} &= -XZ + \\rho X -Y \\\\
\\dot{Z} &= XY - \\beta Z
\\end{aligned}
```
The famous three dimensional system due to Lorenz [1], shown to exhibit
so-called "deterministic nonperiodic flow". It was originally invented to study a
simplified form of atmospheric convection.

Currently, it is most famous for its strange attractor (occuring at the default
parameters), which resembles a butterfly. For the same reason it is
also associated with the term "butterfly effect" (a term which Lorenz himself disliked)
even though the effect applies generally to dynamical systems.
Default values are the ones used in the original paper.

The parameter container has the parameters in the same order as stated in this
function's documentation string.

[1] : E. N. Lorenz, J. atmos. Sci. **20**, pp 130 (1963)
"""
function lorenz(u0=[0.0, 10.0, 0.0]; σ = 10.0, ρ = 28.0, β = 8/3)
    return CDS(loop, u0, [σ, ρ, β], loop_jac)
end
function loop(u, p, t)
    @inbounds begin
        σ = p[1]; ρ = p[2]; β = p[3]
        du1 = σ*(u[2]-u[1])
        du2 = u[1]*(ρ-u[3]) - u[2]
        du3 = u[1]*u[2] - β*u[3]
        return SVector{3}(du1, du2, du3)
    end
end
function loop_jac(u, p, t)
    @inbounds begin
        σ, ρ, β = p
        J = @SMatrix [-σ  σ  0;
        ρ - u[3]  (-1)  (-u[1]);
        u[2]   u[1]  -β]
        return J
    end
end

function lorenz_iip(u0=[0.0, 10.0, 0.0]; σ = 10.0, ρ = 28.0, β = 8/3)
    return CDS(liip, u0, [σ, ρ, β], liip_jac)
end
function liip(du, u, p, t)
    @inbounds begin
        σ = p[1]; ρ = p[2]; β = p[3]
        du[1] = σ*(u[2]-u[1])
        du[2] = u[1]*(ρ-u[3]) - u[2]
        du[3] = u[1]*u[2] - β*u[3]
        return nothing
    end
end
function liip_jac(J, u, p, t)
    @inbounds begin
    σ, ρ, β = p
    J[1,1] = -σ; J[1, 2] = σ; J[1,3] = 0
    J[2,1] = ρ - u[3]; J[2,2] = -1; J[2,3] = -u[1]
    J[3,1] = u[2]; J[3,2] = u[1]; J[3,3] = -β
    return nothing
    end
end

"""
```julia
chua(u0 = [0.7, 0.0, 0.0]; a = 15.6, b = 25.58, m0 = -8/7, m1 = -5/7)
```
```math
\\begin{aligned}
\\dot{x} &= \\alpha (y - h(x))\\\\
\\dot{y} &= x - y+z \\\\
\\dot{z} &= \\beta y
\\end{aligned}
```
where h(x) is defined by
```math
h(x) = m_1 x + \\frac 1 2 (m_0 - m_1)(|x + 1| - |x - 1|)
```
This is a 3D continuous system that exhibits chaos.

Chua designed an electronic circuit with the expressed goal of exhibiting
chaotic motion, and this system is obtained by rescaling the circuit units
to simplify the form of the equation. [1]

The parameters are a, b, m0 and m1. Setting a = 15.6, m0 = -8/7 and m1 = -5/7,
and varying the parameter b from b = 25 to b = 51, one observes a classic
period-doubling bifurcation route to chaos. [2]

The parameter container has the parameters in the same order as stated in this
function's documentation string.

[1] : Chua, Leon O. "The genesis of Chua's circuit", 1992.

[2] : [Leon O. Chua (2007) "Chua circuit", Scholarpedia, 2(10):1488.](http://www.scholarpedia.org/article/Chua_circuit)

"""
function chua(u0 = [0.7, 0.0, 0.0]; a = 15.6, b = 25.58, m0 = -8/7, m1 = -5/7)
    return CDS(chua_rule, u0, [a, b, m0, m1], chua_jacob)
end
function chua_rule(u, p, t)
    @inbounds begin
    a, b, m0, m1 = p
    du1 = a * (u[2] - u[1] - chua_element(u[1], m0, m1))
    du2 = u[1] - u[2] + u[3]
    du3 = -b * u[2]
    return SVector{3}(du1, du2, du3)
    end
end
function chua_jacob(u, p, t)
    a, b, m0, m1 = p
    return @SMatrix[-a*(1 + chua_element_derivative(u[1], m0, m1)) a 0;
                    1 -1 1;
                    0 -b 0]
end
# Helper functions for Chua's circuit.
function chua_element(x, m0, m1)
    return m1 * x + 0.5 * (m0 - m1) * (abs(x + 1.0) - abs(x - 1.0))
end
function chua_element_derivative(x, m0, m1)
    return m1 + 0.5 * (m0 - m1) * (-1 < x < 1 ? 2 : 0)
end

"""
```julia
roessler(u0=[1, -2, 0.1]; a = 0.2, b = 0.2, c = 5.7)
```
```math
\\begin{aligned}
\\dot{x} &= -y-z \\\\
\\dot{y} &= x+ay \\\\
\\dot{z} &= b + z(x-c)
\\end{aligned}
```
This three-dimensional continuous system is due to Rössler [1].
It is a system that by design behaves similarly
to the `lorenz` system and displays a (fractal)
strange attractor. However, it is easier to analyze qualitatively, as for example
the attractor is composed of a single manifold.
Default values are the same as the original paper.

The parameter container has the parameters in the same order as stated in this
function's documentation string.

[1] : O. E. Rössler, Phys. Lett. **57A**, pp 397 (1976)
"""
function roessler(u0=[1, -2, 0.1]; a = 0.2, b = 0.2, c = 5.7)
    return CDS(roessler_rule, u0, [a, b, c], roessler_jacob)
end
function roessler_rule(u, p, t)
    @inbounds begin
    a, b, c = p
    du1 = -u[2]-u[3]
    du2 = u[1] + a*u[2]
    du3 = b + u[3]*(u[1] - c)
    return SVector{3}(du1, du2, du3)
    end
end
function roessler_jacob(u, p, t)
    a, b, c = p
    return @SMatrix [0.0 (-1.0) (-1.0);
                     1.0 a 0.0;
                     u[3] 0.0 (u[1]-c)]
end

"""
    double_pendulum(u0 = [π/2, 0, 0, 0.5];
                    G=10.0, L1 = 1.0, L2 = 1.0, M1 = 1.0, M2 = 1.0)
Famous chaotic double pendulum system (also used for our logo!). Keywords
are gravity (G), lengths of each rod and mass of each ball (all assumed SI units).

The variables order is [θ1, dθ1/dt, θ2, dθ2/dt].

Jacobian is created automatically (thus methods that use the Jacobian will be slower)!

(please contribute the Jacobian and the e.o.m. in LaTeX :smile:)

The parameter container has the parameters in the same order as stated in this
function's documentation string.
"""
function double_pendulum(u0=[π/2, 0, 0, 0.5]; G=10.0, L1 = 1.0, L2 = 1.0, M1 = 1.0, M2 = 1.0)
    return CDS(doublependulum_rule, u0, [G, L1, L2, M1, M2])
end
@inbounds function doublependulum_rule(state, p, t)
    G, L1, L2, M1, M2 = p

    du1 = state[2]
    del_ = state[3] - state[1]
    den1 = (M1 + M2)*L1 - M2*L1*cos(del_)*cos(del_)
    du2 = (M2*L1*state[2]*state[2]*sin(del_)*cos(del_) +
               M2*G*sin(state[3])*cos(del_) +
               M2*L2*state[4]*state[4]*sin(del_) -
               (M1 + M2)*G*sin(state[1]))/den1

    du3 = state[4]

    den2 = (L2/L1)*den1
    du4 = (-M2*L2*state[4]*state[4]*sin(del_)*cos(del_) +
               (M1 + M2)*G*sin(state[1])*cos(del_) -
               (M1 + M2)*L1*state[2]*state[2]*sin(del_) -
               (M1 + M2)*G*sin(state[3]))/den2
    return SVector{4}(du1, du2, du3, du4)
end

"""
    henonheiles(u0=[0, -0.25, 0.42081,0])
```math
\\begin{aligned}
\\dot{x} &= p_x \\\\
\\dot{y} &= p_y \\\\
\\dot{p}_x &= -x -2 xy \\\\
\\dot{p}_y &= -y - (x^2 - y^2)
\\end{aligned}
```

The Hénon–Heiles system [1] is a conservative dynamical system and was introduced as a simplification of the motion
of a star around a galactic center. It was originally intended to study the
existence of a "third integral of motion" (which would make this 4D system integrable).
In that search, the authors encountered chaos, as the third integral existed
for only but a few initial conditions.

The default initial condition is a typical chaotic orbit.
The function `Systems.henonheiles_ics(E, n)` generates a grid of
`n×n` initial conditions, all having the same energy `E`.

[1] : Hénon, M. & Heiles, C., The Astronomical Journal **69**, pp 73–79 (1964)
"""
function henonheiles(u0=[0, -0.25, 0.42081, 0]#=; conserveE::Bool = true=#)
    i = one(eltype(u0))
    o = zero(eltype(u0))
    J = zeros(eltype(u0), 4, 4)
    return CDS(hhrule!, u0, nothing, hhjacob!, J)
end
function hhrule!(du, u, p, t)
    @inbounds begin
        du[1] = u[3]
        du[2] = u[4]
        du[3] = -u[1] - 2u[1]*u[2]
        du[4] = -u[2] - (u[1]^2 - u[2]^2)
        return nothing
    end
end
function hhjacob!(J, u, p, t)
    @inbounds begin
    o = 0.0; i = 1.0
    J[1,:] .= (o,    o,     i,    o)
    J[2,:] .= (o,    o,     o,    i)
    J[3,:] .= (-i - 2*u[2],   -2*u[1],   o,   o)
    J[4,:] .= (-2*u[1],  -1 + 2*u[2],  o,   o)
    return nothing
    end
end
_hhenergy(x,y,px,py) = 0.5(px^2 + py^2) + _hhpotential(x,y)
_hhpotential(x, y) = 0.5(x^2 + y^2) + (x^2*y - (y^3)/3)
function henonheiles_ics(E, n=10)
    ys = range(-0.4, stop = 1.0, length = n)
    pys = range(-0.5, stop = 0.5, length = n)
    ics = Vector{Vector{Float64}}()
    for y in ys
        V = _hhpotential(0.0, y)
        V ≥ E && continue
        for py in pys
            Ky = 0.5*(py^2)
            Ky + V ≥ E && continue
            px = sqrt(2(E - V - Ky))
            ic = [0.0, y, px, py]
            push!(ics, [0.0, y, px, py])
        end
    end
    return ics
end

"""
    qbh([u0]; A=1.0, B=0.55, D=0.4)

A conservative dynamical system with rule
```math
\\begin{aligned}
\\dot{q}_0 &= A p_0 \\\\
\\dot{q}_2 &= A p_2 \\\\
\\dot{p}_0 &= -A q_0 -3 \\frac{B}{\\sqrt{2}} (q_2^2 - q_1^2) - D q_1 (q_1^2 + q_2^2) \\\\
\\dot{p}_2 &= -q_2 (A + 3\\sqrt{2} B q_1 + D (q_1^2 + q_2^2)) (x^2 - y^2)
\\end{aligned}
```

This dynamical rule corresponds to a Hamiltonian used in nuclear
physics to study the quadrupole vibrations of the nuclear surface [1,2].

```math
H(p_0, p_2, q_0, q_2) = \\frac{A}{2}\\left(p_0^2+p_2^2\\right)+\\frac{A}{2}\\left(q_0^2+q_2^2\\right)
			 +\\frac{B}{\\sqrt{2}}q_0\\left(3q_2^2-q_0^2\\right) +\\frac{D}{4}\\left(q_0^2+q_2^2\\right)^2
```

The Hamiltonian has a similar structure with the Henon-Heiles one, but it has an added fourth order term
and presents a nontrivial dependence of chaoticity with the increase of energy [3].
The default initial condition is chaotic.

[1]: Eisenberg, J.M., & Greiner, W., Nuclear theory 2 rev ed. Netherlands: North-Holland pp 80 (1975)

[2]: Baran V. and Raduta A. A., International Journal of Modern Physics E, **7**, pp 527--551 (1998)

[3]: Micluta-Campeanu S., Raportaru M.C., Nicolin A.I., Baran V., Rom. Rep. Phys. **70**, pp 105 (2018)
"""
function qbh(u0=[0., -2.5830294658973876, 1.3873470962626937, -4.743416490252585];  A=1., B=0.55, D=0.4)
    return CDS(qrule, u0, [A, B, D])
end
function qrule(z, p, t)
    @inbounds begin
        A, B, D = p
        p₀, p₂ = z[1], z[2]
        q₀, q₂ = z[3], z[4]

        return SVector{4}(
            -A * q₀ - 3 * B / √2 * (q₂^2 - q₀^2) - D * q₀ * (q₀^2 + q₂^2),
            -q₂ * (A + 3 * √2 * B * q₀ + D * (q₀^2 + q₂^2)),
            A * p₀,
            A * p₂
        )
    end
end

"""
    lorenz96(N::Int, u0 = rand(M); F=0.01)

```math
\\frac{dx_i}{dt} = (x_{i+1}-x_{i-2})x_{i-1} - x_i + F
```

`N` is the chain length, `F` the forcing. Jacobian is created automatically.
(parameter container only contains `F`)
"""
function lorenz96(N::Int, u0 = range(0; length = N, step = 0.1); F=0.01)
    @assert N ≥ 4 "`N` must be at least 4"
    lor96 = Lorenz96{N}() # create struct
    return CDS(lor96, u0, [F])
end
struct Lorenz96{N} end # Structure for size type
function (obj::Lorenz96{N})(dx, x, p, t) where {N}
    F = p[1]
    # 3 edge cases
    @inbounds dx[1] = (x[2] - x[N - 1]) * x[N] - x[1] + F
    @inbounds dx[2] = (x[3] - x[N]) * x[1] - x[2] + F
    @inbounds dx[N] = (x[1] - x[N - 2]) * x[N - 1] - x[N] + F
    # then the general case
    for n in 3:(N - 1)
      @inbounds dx[n] = (x[n + 1] - x[n - 2]) * x[n - 1] - x[n] + F
    end
    return nothing
end

"""
    duffing(u0 = [0.1, 0.25]; ω = 2.2, f = 27.0, d = 0.2, β = 1)
The (forced) duffing oscillator, that satisfies the equation
```math
\\ddot{x} + d\\cdot\\dot{x} + β*x + x^3 = f\\cos(\\omega t)
```
with `f, ω` the forcing strength and frequency and `d` the dampening.

The parameter container has the parameters in the same order as stated in this
function's documentation string.
"""
function duffing(u0 = [0.1, 0.25]; ω = 2.2, f = 27.0, d = 0.2, β = 1)
    J = zeros(eltype(u0), 2, 2)
    J[1,2] = 1
    return CDS(duffing_rule, u0, [ω, f, d, β], duffing_jacob, J)
end
@inbounds function duffing_rule(x, p, t)
    ω, f, d, β = p
    dx1 = x[2]
    dx2 = f*cos(ω*t) - β*x[1] - x[1]^3 - d * x[2]
    return SVector(dx1, dx2)
end
@inbounds function duffing_jacob(u, p, t)
    ω, f, d, β = p
    return @SMatrix [0 1 ;
    (-β - 3u[1]^2) -d]
end

"""
    shinriki(u0 = [-2, 0, 0.2]; R1 = 22.0)
Shinriki oscillator with all other parameters (besides `R1`) set to constants.
*This is a stiff problem, be careful when choosing solvers and tolerances*.
"""
function shinriki(u0 = [-2, 0, 0.2]; R1 = 22.0)
    # # Jacobian caller for Shinriki:
    # shinriki_rule(::Type{Val{:jac}}, J, u, p, t) = (shi::Shinriki)(t, u, J)
    return CDS(shinriki_rule, u0, [R1])
end
shinriki_voltage(V) = 2.295e-5*(exp(3.0038*V) - exp(-3.0038*V))
function shinriki_rule(u, p, t)
    R1 = p[1]

    du1 = (1/0.01)*(
    u[1]*(1/6.9 - 1/R1) - shinriki_voltage(u[1] - u[2]) - (u[1] - u[2])/14.5
    )

    du2 = (1/0.1)*(
    shinriki_voltage(u[1] - u[2]) + (u[1] - u[2])/14.5 - u[3]
    )

    du3 = (1/0.32)*(-u[3]*0.1 + u[2])
    return SVector{3}(du1, du2, du3)
end

"""
```julia
gissinger(u0 = [3, 0.5, 1.5]; μ = 0.119, ν = 0.1, Γ = 0.9)
```
```math
\\begin{aligned}
\\dot{Q} &= \\mu Q - VD \\\\
\\dot{D} &= -\\nu D + VQ \\\\
\\dot{V} &= \\Gamma -V + QD
\\end{aligned}
```
A continuous system that models chaotic reversals due to Gissinger [1], applied
to study the reversals of the magnetic field of the Earth.

The parameter container has the parameters in the same order as stated in this
function's documentation string.

[1] : C. Gissinger, Eur. Phys. J. B **85**, 4, pp 1-12 (2012)
"""
function gissinger(u0 = [3, 0.5, 1.5]; μ = 0.119, ν = 0.1, Γ = 0.9)
    return CDS(gissinger_rule, u0, [μ, ν, Γ], gissinger_jacob)
end
function gissinger_rule(u, p, t)
    μ, ν, Γ = p
    du1 = μ*u[1] - u[2]*u[3]
    du2 = -ν*u[2] + u[1]*u[3]
    du3 = Γ - u[3] + u[1]*u[2]
    return SVector{3}(du1, du2, du3)
end
function gissinger_jacob(u, p, t)
    μ, ν, Γ = p
    return @SMatrix [μ -u[3] -u[2];
                     u[3] -ν u[1];
                     u[2] u[1] -1]
end

"""
```julia
rikitake(u0 = [1, 0, 0.6]; μ = 1.0, α = 1.0)
```
```math
\\begin{aligned}
\\dot{x} &= -\\mu x +yz \\\\
\\dot{y} &= -\\mu y +x(z-\\alpha) \\\\
\\dot{z} &= 1 - xz
\\end{aligned}
```
Rikitake's dynamo is a system that tries to model the magnetic reversal events
by means of a double-disk dynamo system.

[1] : T. Rikitake Math. Proc. Camb. Phil. Soc. **54**, pp 89–105, (1958)
"""
function rikitake(u0 = [1, 0, 0.6]; μ = 1.0, α = 1.0)
    return CDS(rikitake_rule, u0, [μ, α], rikitake_jacob)
end
function rikitake_rule(u, p, t)
    μ, α = p
    x,y,z = u
    xdot = -μ*x + y*z
    ydot = -μ*y + x*(z - α)
    zdot = 1 - x*y
    return SVector{3}(xdot, ydot, zdot)
end
function rikitake_jacob(u, p, t)
    μ, α = p
    x,y,z = u
    xdot = -μ*x + y*z
    ydot = -μ*y + x*(z - α)
    zdot = 1 - x*y
    return @SMatrix [-μ  z  y;
                     z-α -μ x;
                     -y  -x 0]
end

"""
```julia
nosehoover(u0 = [0, 0.1, 0])
```
```math
\\begin{aligned}
\\dot{x} &= y \\\\
\\dot{y} &= yz - x \\\\
\\dot{z} &= 1 - y^2
\\end{aligned}
```
Three dimensional conservative continuous system, discovered in 1984 during
investigations in thermodynamical chemistry by Nosé and Hoover, then
rediscovered by Sprott during an exhaustive search as an extremely simple
chaotic system. [1]

See Chapter 4 of "Elegant Chaos" by J. C. Sprott. [2]

[1] : Hoover, W. G. (1995). Remark on ‘‘Some simple chaotic flows’’. *Physical Review E*, *51*(1), 759.

[2] : Sprott, J. C. (2010). *Elegant chaos: algebraically simple chaotic flows*. World Scientific.
"""
nosehoover(u0 = [0, 0.1, 0]) = CDS(nosehoover_rule, u0, nothing, nosehoover_jacob)
function nosehoover_rule(u, p, t)
    x,y,z = u
    xdot = y
    ydot = y*z - x
    zdot  = 1.0 - y*y
    return SVector{3}(xdot, ydot, zdot)
end
function nosehoover_jacob(u, p, t)
    x,y,z = u
    return @SMatrix [0 1 0;
                     -1 z y;
                     0 2y 0]
end

"""
    antidots([u]; B = 1.0, d0 = 0.3, c = 0.2)
An antidot "superlattice" is a Hamiltonian system that corresponds to a
smoothened periodic Sinai billiard with disk diameter `d0` and smooth
factor `c` [1].

This version is the two dimensional
classical form of the system, with quadratic dynamical rule and
a perpendicular magnetic field. Notice that the dynamical rule
is with respect to the velocity instead of momentum, i.e.:
```math
\\begin{aligned}
\\dot{x} &= v_x \\\\
\\dot{y} &= v_y \\\\
\\dot{v_x} &= B*v_y - U_x \\\\
\\dot{v_y} &= -B*v_x - U_X \\\\
\\end{aligned}
```
with ``U`` the potential energy:
```math
U = \\left(\\tfrac{1}{c^4}\\right) \\left[\\tfrac{d_0}{2} + c - r_a\\right]^4
```
if ``r_a = \\sqrt{(x \\mod 1)^2 + (y \\mod 1)^2} < \\frac{d_0}{2} + c`` and 0
otherwise. I.e. the potential is periodic with period 1 in both ``x, y`` and
normalized such that for energy value of 1 it is a circle of diameter ``d0``.
The magnetic field is also normalized such that for value `B=1` the cyclotron
diameter is 1.

[1] : G. Datseris *et al*, [New Journal of Physics 2019](https://iopscience.iop.org/article/10.1088/1367-2630/ab19cc/meta)
"""
function antidots(u0 = [0.5, 0.5, 0.25, 0.25];
    d0 = 0.5, c = 0.2, B = 1.0)
    return CDS(antidot_rule, u0, [B, d0, c], antidot_jacob)
end

function antidot_rule(u, p, t)
    B, d0, c = p
    x, y, vx, vy = u
    # Calculate quadrant of (x,y):
    U = Uy = Ux = 0.0
    xtilde = x - round(x);  ytilde = y - round(y)
    ρ = sqrt(xtilde^2 + ytilde^2)
    # Calculate derivatives and potential:
    if ρ < 0.5*d0 + c
        sharedfactor = -(4*(c + d0/2 - ρ)^(3))/(c^4*ρ)
        Ux = sharedfactor*xtilde # derivatives
        Uy = sharedfactor*ytilde
    end
    Br = 2*√(2)*B # magnetic field with prefactor
    return SVector{4}(vx, vy, Br*vy - Ux, -vx*Br - Uy)
end
function antidot_jacob(u, p, t)
    B, d0, c = p
    x, y, vx, vy = u
    xtilde = x - round(x);  ytilde = y - round(y)
    ρ = sqrt(xtilde^2 + ytilde^2)
    # Calculate derivatives and potential:
    if ρ < 0.5*d0 + c
        Uxx, Uyy, Uxy = antidot_secondderv(xtilde, ytilde, p)
    else
        Uxx, Uyy, Uxy = 0.0, 0.0, 0.0
    end
    Br = 2*√(2)*B # magnetic field with prefactor
    return SMatrix{4, 4}(0.0, 0, -Uxx, -Uxy, 0, 0, -Uxy, -Uyy,
                         1, 0, 0, -Br,        0, 1, Br, 0)
end

function antidot_potential(x::Real, y::Real, p)
    B, d0, c = p
    δ = 4
    # Calculate quadrant of (x,y):
    xtilde = x - round(x);  ytilde = y - round(y)
    ρ = sqrt(xtilde^2 + ytilde^2)
    # Check distance:
    pot = ρ > 0.5*d0 + c ? 0.0 : (1/(c^δ))*(0.5*d0 + c - ρ)^δ
end

function antidot_secondderv(x, y, p)
    B, d0, c = p
    r = sqrt(x^2 + y^2)
    Uxx = _Uxx(x, y, c, d0, r)
    Uyy = _Uxx(y, x, c, d0, r)
    Uxy =  (2c + d0 - 2r)^2 * (2c + d0 + 4r)*x*y / (2c^4*r^3)
    return Uxx, Uyy, Uxy
end

function _Uxx(x, y, c, d0, r)
    Uxx =  (2c + d0 - 2r)^2 * (-2c*y^2 - d0*y^2 + 2*r*(3x^2 + y^2)) /
    (2 * (c^4) * r^3)
end

"""
```julia
ueda(u0 = [3.0, 0]; k = 0.1, B = 12.0)
```
```math
\\ddot{x} + k \\dot{x} + x^3 = B\\cos{t}
```
Nonautonomous Duffing-like forced oscillation system, discovered by Ueda in
1961. It is one of the first chaotic systems to be discovered.

The stroboscopic plot in the (x, ̇x) plane with period 2π creates a "broken-egg
attractor" for k = 0.1 and B = 12. Figure 5 of [1] is reproduced by

```julia
using Plots
ds = Systems.ueda()
a = trajectory(ds, 2π*5e3, dt = 2π)
scatter(a[:, 1], a[:, 2], markersize = 0.5, title="Ueda attractor")
```

For more forced oscillation systems, see Chapter 2 of "Elegant Chaos" by
J. C. Sprott. [2]

[1] : [Ruelle, David, ‘Strange Attractors’, The Mathematical Intelligencer, 2.3 (1980), 126–37](https://doi.org/10/dkfd3n)

[2] : Sprott, J. C. (2010). *Elegant chaos: algebraically simple chaotic flows*. World Scientific.
"""
function ueda(u0 = [3.0, 0]; k = 0.1, B = 12.0)
    return CDS(ueda_rule, u0, [k, B], ueda_jacob)
end
function ueda_rule(u, p, t)
    x,y = u
    k, B = p
    xdot = y
    ydot = B*cos(t) - k*y - x^3
    return SVector{2}(xdot, ydot)
end
function ueda_jacob(u, p, t)
    x,y = u
    k, B = p
    return @SMatrix [0      1;
                     -3*x^2 -k]
end

struct MagneticPendulum
    magnets::Vector{SVector{2, Float64}}
end
mutable struct MagneticPendulumParams
    γs::Vector{Float64}
    d::Float64
    α::Float64
    ω::Float64
end

function (m::MagneticPendulum)(u, p, t)
    x, y, vx, vy = u
    γs::Vector{Float64}, d::Float64, α::Float64, ω::Float64 = p.γs, p.d, p.α, p.ω
    dx, dy = vx, vy
    dvx, dvy = @. -ω^2*(x, y) - α*(vx, vy)
    for (i, ma) in enumerate(m.magnets)
        δx, δy = (x - ma[1]), (y - ma[2])
        D = sqrt(δx^2 + δy^2 + d^2)
        dvx -= γs[i]*(x - ma[1])/D^3
        dvy -= γs[i]*(y - ma[2])/D^3
    end
    return SVector(dx, dy, dvx, dvy)
end

"""
    magnetic_pendulum(u=[0.7,0.7,0,0]; d=0.3, α=0.2, ω=0.5, N=3, γs=fill(1.0,N))

Create a pangetic pendulum with `N` magnetics, equally distributed along the unit circle,
with dynamical rule
```math
\\begin{aligned}
\\ddot{x} &= -\\omega ^2x - \\alpha \\dot{x} - \\sum_{i=1}^N \\frac{\\gamma_i (x - x_i)}{D_i^3} \\\\
\\ddot{y} &= -\\omega ^2y - \\alpha \\dot{y} - \\sum_{i=1}^N \\frac{\\gamma_i (y - y_i)}{D_i^3} \\\\
D_i &= \\sqrt{(x-x_i)^2  + (y-y_i)^2 + d^2}
\\end{aligned}
```
where α is friction, ω is eigenfrequency, d is distance of pendulum from the magnet's plane
and γ is the magnetic strength.
"""
function magnetic_pendulum(u = [sincos(0.12553*2π)..., 0, 0];
    γ = 1.0, d = 0.3, α = 0.2, ω = 0.5, N = 3, γs = fill(γ, N))
    m = MagneticPendulum([SVector(cos(2π*i/N), sin(2π*i/N)) for i in 1:N])
    p = MagneticPendulumParams(γs, d, α, ω)
    ds = ContinuousDynamicalSystem(m, u, p)
end

"""
    fitzhugh_nagumo(u = 0.5ones(2); a=3.0, b=0.2, ε=0.01, I=0.0)
Famous excitable system which emulates the firing of a neuron, with rule
```math
\\begin{aligned}
\\dot{v} &= av(v-b)(1-v) - w + I \\\\
\\ddot{w} &= \\varepsilon(v - w)
\\end{aligned}
```

More details in the [Scholarpedia](http://www.scholarpedia.org/article/FitzHugh-Nagumo_model) entry.
"""
function fitzhugh_nagumo(u = 0.5ones(2); a=3.0, b=0.2, ε=0.01, I=0.0)
    ds = ContinuousDynamicalSystem(fitzhugh_nagumo_rule, u, [a, b, ε, I])
end
function fitzhugh_nagumo_rule(x, p, t)
    u, w = x
    a, b, ε, I = p
    return SVector(a*u*(u-b)*(1. - u) - w + I, ε*(u - w))
end

"""
    more_chaos_example(u = rand(3))
A three dimensional chaotic system introduced in [^Sprott2020] with rule
```math
\\begin{aligned}
\\dot{x} &= y \\\\
\\dot{y} &= -x - sign(z)y \\\\
\\dot{z} &= y^2 - \\exp(-x^2)
\\end{aligned}
```
It is noteworthy because its strange attractor is multifractal with fractal dimension ≈ 3.

[^Sprott2020]: Sprott, J.C. 'Do We Need More Chaos Examples?', Chaos Theory and Applications 2(2),1-3, 2020
"""
more_chaos_example(u = [0.0246, 0.79752, 0.3535866]) =
ContinuousDynamicalSystem(more_chaos_rule, u, nothing)
function more_chaos_rule(u, p, t)
    x, y, z = u
    dx = y
    dy = -x - sign(z)*y
    dz = y^2 - exp(-x^2)
    return SVector(dx, dy, dz)
end

"""
    thomas_cyclical(u0 = [1.0, 0, 0]; b = 0.2)
```math
\\begin{aligned}
\\dot{x} &= \\sin(y) - bx\\\\
\\dot{y} &= \\sin(z) - by\\\\
\\dot{z} &= \\sin(x) - bz
\\end{aligned}
```
Thomas' cyclically symmetric attractor is a 3D strange attractor originally proposed
by René Thomas[^Thomas1999]. It has a simple form which is cyclically symmetric in the
x,y, and z variables and can be viewed as the trajectory of a frictionally dampened
particle moving in a 3D lattice of forces.
For more see the [Wikipedia page](https://en.wikipedia.org/wiki/Thomas%27_cyclically_symmetric_attractor).

Reduces to the labyrinth system for `b=0`, see
See discussion in Section 4.4.3 of "Elegant Chaos" by J. C. Sprott.

[^Thomas1999]: Thomas, R. (1999). *International Journal of Bifurcation and Chaos*, *9*(10), 1889-1905.
"""
thomas_cyclical(u0 = [1.0, 0, 0]; b = 0.2) = CDS(thomas_rule, u0, [b], thomas_jacob)
labyrinth(u0 = [1.0, 0, 0]) = CDS(thomas_rule, u0, [0.0], thomas_jacob)

function thomas_rule(u, p, t)
    x,y,z = u
    b = p[1]
    xdot = sin(y) - b*x
    ydot = sin(z) - b*y
    zdot = sin(x) - b*z
    return SVector{3}(xdot, ydot, zdot)
end
function thomas_jacob(u, p, t)
    x,y,z = u
    b = p[1]
    return @SMatrix [b cos(y) 0;
                     0 b cos(z);
                     cos(x) 0 b]
end

"""
    stommel_thermohaline(u = [0.3, 0.2]; η1 = 3.0, η2 = 1, η3 = 0.3)
Stommel's box model for Atlantic thermohaline circulation
```math
\\begin{aligned}
 \\dot{T} &= \\eta_1 - T - |T-S|\\cdot T \\\\
 \\dot{S} &= \\eta_2 - \\eta_3S - |T-S|\\cdot S
\\end{aligned}
```
Here ``T, S`` are variables standing for dimensionless temperature and salinity
differences between the boxes (polar and equitorial ocean basins) and ``\\eta_i``
are parameters.

[^Stommel1961]: Stommel, Thermohaline convection with two stable regimes offlow. Tellus, 13(2)
"""
function stommel_thermohaline(u = [0.3, 0.2]; η1 = 3.0, η2 = 1, η3 = 0.3)
    ds = ContinuousDynamicalSystem(stommel_thermohaline_rule, u, [η1, η2, η3],
    stommel_thermohaline_jacob)
end
function stommel_thermohaline_rule(x, p, t)
    T, S = x
    η1, η2, η3 = p
    q = abs(T-S)
    return SVector(η1 - T -q*T, η2 - η3*S - q*S)
end
function stommel_thermohaline_jacob(x, p, t)
    T, S = x
    η1, η2, η3 = p
    q = abs(T-S)
    if T ≥ S
        return @SMatrix [(-1 - 2T + S)  (T);
                         (-S)  (-η3 - T + 2S)]
    else
        return @SMatrix [(-1 + 2T - S)  (-T);
                         (+S)  (-η3 + T - 2S)]
    end
end


using Images, LinearAlgebra, Plots

function low_rank_approx(x::SVD{T}, k) where T <: Real
    #take a low rank (k) approx of the SVD of a matrix
    #via outer products
    return x.U[:, 1:k] * Diagonal(x.S[1:k]) * x.V[:, 1:k]'
end

image_path = joinpath(dirname(@__DIR__), "data/20210501_171236.jpg")

img_big = load(image_path)
img = Images.imresize(img_big, ratio = 1/4)

channels = channelview(img)
svd_r = svd(channels[1,:,:])
svd_g = svd(channels[2,:,:])
svd_b = svd(channels[3,:,:])

p1 = plot(svd_r.S, c = :red, label = "red singular values")
plot!(p1, svd_g.S, c = :green, label = "green singular values")
plot!(p1, svd_b.S, c = :blue, label = "blue singular values")

#recall trace norm identity

# ||A||f  = √sum(σ(A)^2)

#Frobenius norm => Pretend matrix is big vector and take l2 norm
# hints that truncated svd might contain a good representation on matrix

#in case you don't trust me...
norm(channels[1,:,:]) ≈ norm(svd_r.S)
norm(channels[2,:,:]) ≈ norm(svd_g.S)
norm(channels[3,:,:]) ≈ norm(svd_b.S)

approx_error = [norm(low_rank_approx(svd_r, i) - channels[1,:,:]) for i = 1 :10: length(svd_r.S)]

p2 = plot(approx_error./norm(channels[1,:,:]))

#what does it look like though...

reconstructed_images = []

for k = 1:10:60
    r_low = low_rank_approx(svd_r, k)
    g_low = low_rank_approx(svd_g, k)
    b_low = low_rank_approx(svd_b, k)
    push!(reconstructed_images, colorview(RGB, r_low, g_low, b_low))
end

output_plot = [plot(ri) for ri in reconstructed_images]

plot(output_plot...)

#So we can get a linear reduction in size by keeping the components of the
#outer product U[:, 1:k], S[1:k], V[:, 1:k] and reconstructing via matrix
#multiplication

#So...

@show sizeof(img_big)/1e6

#So to get approx 1% reconstruction error in Frobenius norm we need about half the singular values

percent_reduction_01 = findfirst(approx_error./norm(channels[1,:,:]) .< 0.01)/length(approx_error)

@show percent_reduction_01

#so the final "saved" image would be

@show (sizeof(img_big)/1e6) * percent_reduction_01

#not bad, but look at what jpeg gets...


function linear_kernel(x1::Vector, x2::Vector)
  return dot(x1, x2)::Float64
end

function polynomial_kernel(x1::Vector, x2::Vector, d::Integer=2)
  return ((1 + dot(x1, x2)) ^ d)::Float64
end

# function gaussian_kernel(x1::Vector, x2::Vector, sigma::Real=1.0)
#   # k = exp(-1/(2*sigma^2)  ||x1 - x2||^2 )
#   return exp( -norm(x1 - x2)^2 / (2*sigma^2) )::Float64
# end

function gaussian_kernel(x1::Vector, x2::Vector, gamma::Real=1.0)
  # k = exp(-gamma  ||x1 - x2||^2 )
  return exp( -gamma * norm(x1 - x2)^2 )::Float64
end


module poly
using LinearAlgebra

# Base.+ OK
# Base.- OK
# Base.* OK
# degree OK
# pow to a positive integer OK
# eval OK
# derivative OK
# integrate OK
# roots OK
# create polynomial from roots OK

struct Polynomial{N, T}
    coeff::NTuple{N, T}

    function Polynomial{N, T}(coeff::NTuple{N, T}) where {N, T}
        return new(coeff)
    end
end

function Polynomial(coeff::AbstractArray{T}) where {T}
    N = length(coeff)
    t = tuple(coeff...)
    return Polynomial{N, T}(t)
end

function Polynomial(coeff::Tuple)
    t = promote(coeff...)
    N = length(coeff)
    T = eltype(t)
    return Polynomial{N,T}(t)
    # return Polynomial(t)
end

function Polynomial(coeff::T) where {T <: Number}
    return Polynomial((coeff,))
end

function Base.eltype(p::Polynomial)
    return eltype(p.coeff)
end

function degree(p::Polynomial)
    return length(p.coeff)-1
end

function vector(p::Polynomial)
    N = degree(p)+1
    v = Vector{eltype(p)}(undef, N)
    for ii = 1:N
        v[ii] = p.coeff[ii]
    end
    return v
end

function Base.:+(p1::Polynomial, p2::Polynomial)
    N1 = degree(p1)+1
    N2 = degree(p2)+1
    N3 = max(N1, N2)
    T1 = eltype(p1)
    T2 = eltype(p2)
    T3 = promote_type(T1, T2)

    coeff = zeros(T3, N3)
    for ii = 1:N1
        coeff[ii] += p1.coeff[ii]
    end
    for ii = 1:N2
        coeff[ii] += p2.coeff[ii]
    end

    p3 = Polynomial(coeff)
    return p3
end

function Base.:-(p1::Polynomial, p2::Polynomial)
    N1 = degree(p1)+1
    N2 = degree(p2)+1
    N3 = max(N1, N2)
    T1 = eltype(p1)
    T2 = eltype(p2)
    T3 = promote_type(T1, T2)

    coeff = zeros(T3, N3)
    for ii = 1:N1
        coeff[ii] += p1.coeff[ii]
    end
    for ii = 1:N2
        coeff[ii] -= p2.coeff[ii]
    end

    p3 = Polynomial(coeff)
    return p3
end

function Base.:*(p1::Polynomial, p2::Polynomial)

    d1 = degree(p1)
    d2 = degree(p2)
    N1 = d1 + 1
    N2 = d2 + 1
    T1 = eltype(p1)
    T2 = eltype(p2)
    T = promote_type(T1, T2)
    N = d1 + d2 + 1
    v = Vector{T}(undef, N)

    if N1 <= N2
        v1 = p1.coeff
        v2 = p2.coeff
    else
        v1 = p2.coeff
        v2 = p1.coeff
        N1, N2 = N2, N1
    end

    for ii = 1:d1
        sum = 0.0
        for jj = 1:ii
            sum += v1[ii - jj + 1] * v2[jj]
        end
        v[ii] = sum
    end

    for ii = N1:-1:2
        sum = 0.0
        for jj = ii:N1
            sum += v1[jj] * v2[ii - jj + N2]
        end
        v[ii - N1 + N] = sum
    end

    for ii = N1:N2
        sum = 0.0
        for jj = 1:N1
            sum += v1[N1 - jj + 1] * v2[jj + ii - N1]
        end
        v[ii] = sum
    end

    q = Polynomial(v)
    return q
end

function Base.:^(p::Polynomial, n::Int)
    if n < 0
        error("This is not a polynomial any more")
    end
    if n == 0
        q = Polynomial(1.0)
        return q
    end

    q = p
    for ii = 1:(n-1)
        q = q * p
    end
    return q
end

function integrate(p::Polynomial, C = 0.0)
    # int x^m = x^(m+1) / (m+1)

    v = [C, vector(p)...]

    for ii = 2:length(v)
        v[ii] /= ii - 1
    end

    q = Polynomial(v)
    return q
end

function derivative(p::Polynomial)
    # der x^m = m * x^(m-1)

    v = vector(p)
    v = v[2:end]

    for ii = eachindex(v)
        v[ii] *= ii
    end

    q = Polynomial(v)
    return q
end

function highest_nonzero_degree(p::Polynomial; abs_tol = 0.0, rel_tol = eps(1.0))
    d = degree(p)
    max_degree = 0
    for ii = (d+1):-1:1
        if !isapprox(p.coeff[ii], zero(eltype(p)), atol = abs_tol, rtol = rel_tol)
            max_degree = ii - 1
            break
        end
    end

    return max_degree
end

function companion_matrix(p::Polynomial)
    hd = highest_nonzero_degree(p)
    m = zeros(hd, hd)
    for ii = 1:(hd-1)
        m[ii+1, ii] = 1.0
    end
    for ii = 1:hd
        m[1, ii] = - p.coeff[hd + 1 - ii] / p.coeff[hd+1]
    end

    return m
end

function roots(p::Polynomial)
    m = companion_matrix(p)
    F = eigen(m)
    z = F.values
    return z
end

function poly_from_roots(roots::AbstractArray; make_real::Bool = true)
    t = tuple(roots...)
    return poly_from_roots(t; make_real = make_real)
end

function poly_from_roots(roots::Tuple; make_real::Bool = true)
    L = length(roots)
    q = Polynomial(1.0)
    for ii = 1:L
        p = Polynomial( (-roots[ii], one(eltype(roots[1]))) )
        # q = q * p
        q = p * q
    end
    if make_real
        q_real_or_imag = Polynomial( map(real, q.coeff) )
    else
        q_real_or_imag = q
    end
    return q_real_or_imag
end

function poly_eval(p::Polynomial, x)
    return poly_eval(x, p.coeff)
end

function (p::Polynomial)(x)
    return poly_eval(p, x)
end

function poly_eval(x, a0, a1, a2, a3)
    return ((a3 .* x .+ a2) .* x + a1) .* x .+ a0
end

function poly_eval(x, a0, a1, a2)
    return (a2 .* x .+ a1) .* x + a0
end

function poly_eval(x, a0, a1)
    return a1 .* x .+ a0
end

function poly_eval(x::AbstractArray, p)
    # p array polynomial coeffiecents
    # p[1] + p[2] * x + p[3] * x^2 + ...
    y = zeros( length(x) )
    np = length(p)
    if np == 0
        return y
    end
    if np == 1
        # return y .* p[1]
        return p[1] .* ones(np)
    end

    y .= p[np]
    for ii = np:-1:2
        y = y .* x .+ p[ii-1]
    end

    return y
end

function poly_eval(x::T, p) where {T <: Number}
    y = zero(T)
    np = length(p)
    if np == 0
        return y
    end
    if np == 1
        return p[1]
    end

    y = p[np]
    for ii = np:-1:2
        y = y * x + p[ii-1]
    end

    return y
end

end

module ptest
using ..poly

# coeff = (1.0, 2.0, 3.0)
# # t_coeff = tuple(map(Float64, v_coeff)...)

# p_v = poly.Polynomial(v_coeff)
# p_t = poly.Polynomial(t_coeff)

coeff = (1.0, 2.0, 3.0, 5.0)
p = poly.Polynomial(coeff)
int_p = poly.integrate(p)
der_p = poly.derivative(p)

coeff1 = (3.0, 2.0, 4.0)
p1 = poly.Polynomial(coeff1)

coeff2 = (5.0, 6.0, 3.0, 7.0)
p2 = poly.Polynomial(coeff2)

# q = p1 * p2

# p1_sq = p1^2
# p2_sq = p2^2

# z = poly.roots(p1)
# q = poly.poly_from_roots(z)

z = [-3.0, 1.0, 5.0, 2.0 - im, 2.0 + im]
q = poly.poly_from_roots(z)
z_c = poly.roots(q)

end


#===============================================================================

Let r be the remainder when (a−1)^n + (a+1)^n is divided by a^2.

For example, if a = 7 and n = 3, then r = 42: 6^3 + 8^3 = 728 ≡ 42 mod 49. And
as n varies, so too will r, but for a = 7 it turns out that r_max = 42.

For 3 ≤ a ≤ 1000, find ∑ r_max.

--------------------------------------------------------------------------------

Let r(a,n) := ((a−1)^n + (a+1)^n) % a^2
It comes: r(a, n) = 0 for a=1 (because of the modulo 1)
                  = 2 for n even
                  = 2an % a^2 = a(2n % a) for n odd

          max_n(2n % a) = a-1 if a is odd
                        = a-2 if a is even

This max_n(r(a, n)) = 0 for a = 1
                    = 2 for a = 2
                    = a(a-1) for 3 <= a and a odd
                    = a(a-2) for 3 <= a and a even

===============================================================================#

rmax(a) = isodd(a) ? a*(a-1) : a*(a-2) # It is assumed that a>=3

function compute(limit)
    result = 0
    for a in 3:limit
        result += rmax(a)
    end
    result;
end

println("Euler 120: $(compute(1000))")


# ------------------------------------------------------------------------------------------
# # ModInt: a simple modular integer type
# ------------------------------------------------------------------------------------------

struct ModInt{n} <: Integer
    k::Int

    # Constructor definition...
    # note the left side looks like the call it defines
    ModInt{n}(k::Int) where {n} = new(mod(k,n))
end

a = ModInt{13}(1238279873492834)

b = ModInt{13}(9872349827349827)

a + b

# ------------------------------------------------------------------------------------------
# To extend standard functions we need to import them.
# ------------------------------------------------------------------------------------------

import Base: +

+(a::ModInt{n}, b::ModInt{n}) where {n} = ModInt{n}(a.k + b.k)

a + b

import Base: *, -

*(a::ModInt{n}, b::ModInt{n}) where{n} = ModInt{n}(a.k * b.k)
-(a::ModInt{n}, b::ModInt{n}) where {n} = ModInt{n}(a.k - b.k)
-(a::ModInt{n}) where {n} = ModInt{n}(-a.k)

a * b

a - b

-b

Base.show(io::IO, a::ModInt{n}) where {n} =
    get(io, :compact, false) ? show(io, a.k) : print(io, "$(a.k) mod $n")

a

b

a + 1

Base.promote_rule(::Type{ModInt{n}}, ::Type{Int}) where {n} = ModInt{n}

Base.convert(::Type{ModInt{n}}, x::Int) where {n} = ModInt{n}(x)

a + 1

1 + a

A = map(ModInt{13}, rand(1:100, 5, 5))

A^100000000

2A^100 .- 1

# ------------------------------------------------------------------------------------------
# ### Summary
#
# Here is all the code that defines the `ModInt` type:
# ```jl
# struct ModInt{n} <: Integer
#     k::Int
#
#     # Constructor definition...
#     # note the left side looks like the call it defines
#     ModInt{n}(k::Int) where {n} = new(mod(k,n))
# end
#
# import Base: +, *, -
#
# +(a::ModInt{n}, b::ModInt{n}) where {n} = ModInt{n}(a.k + b.k)
# *(a::ModInt{n}, b::ModInt{n}) where{n} = ModInt{n}(a.k * b.k)
# -(a::ModInt{n}, b::ModInt{n}) where {n} = ModInt{n}(a.k - b.k)
# -(a::ModInt{n}) where {n} = ModInt{n}(-a.k)
#
# Base.show(io::IO, a::ModInt{n}) where {n} =
#     get(io, :compact, false) ? show(io, a.k) : print(io, "$(a.k) mod $n")
#
# Base.promote_rule(::Type{ModInt{n}}, ::Type{Int}) where {n} = ModInt{n}
# Base.convert(::Type{ModInt{n}}, x::Int) where {n} = ModInt{n}(x)
# ```
# ------------------------------------------------------------------------------------------

# ------------------------------------------------------------------------------------------
# ### Exercise
#
# Add two methods that allows operations between modular integers with different modulus
# using the rule that they should combine in the modulus that is the `lcm` (least common
# multiple) of the moduli of the arguments.
#
# **Hint:** try something, see what fails, define something to make that work.
# ------------------------------------------------------------------------------------------

x = ModInt{12}(9)

y = ModInt{15}(13)

# two method definitions here...

@assert x + y == ModInt{60}(22)
@assert x * y == ModInt{60}(57)


module Utils

function is_prime(n::Int)
    if n == 2
        return true
    else
        for i = 2:Int(ceil(sqrt(n)))
            if n%i==0
                return false
            end
        end
        return true
    end
end

function next_prime(n::Int)
    i = n+1
    while !is_prime(i)
        i += 1
    end
    return i
end

# Prime number iterator.
struct Primes end
Base.start(::Primes) = 2
Base.next(P::Primes, state) = (state, next_prime(state))
Base.done(P::Primes, state) = false

function factorize(n::Int)
    # Factorizes n = p1^(e1) * p2^(e2) * ... * pk^(ek)
    # into array of tuples [(p1,e1),(p2,e2),...,(pk,ek)]
    # We'll do this by iteratively dividing n by ever
    # larger primes (and powers thereof) until n becomes 1,
    # at which point we'll know factorization is complete.
    factors = []

    for prime in Primes()
        if n % prime == 0 # We've found a factor
            # Find the power to which the prime
            # factor is raised by progressively dividing
            # n by the factor until it no longer divides
            # evenly.
            power = 0
            while n % prime == 0
                n /= prime
                power += 1
            end
            push!(factors, (prime, power))
        end
        if n == 1
            break # We've fully factored n
        end
    end
    return factors
end

end


"""
Sigmoid Activation function
"""
function sigmoid(Z::Array{BigFloat})
    A = max.(min.(1 ./ (1 .+ exp.(.-Z)), 1-eps(1.0)), eps(1.0))
    return (A = A, Z = Z)
end # function

"""
ReLU Activation function
"""
function relu(Z)
    A = max.(0, Z)
    return (A = A, Z = Z)
end # function

"""
Softmax Activation function
"""
function softmax(Z::Array{BigFloat})
    A = max.(exp.(Z) ./ ( sum(exp.(Z))), eps(1.0))
    return (A = A, Z = Z)
end # function

"""
Tanh Activation function
"""
function tanhact(Z::Array{BigFloat})
    A = min.(max.(tanh.(Z), -1.0+eps(1.0)), 1.0-eps(1.0))
    return (A = A, Z = Z)
end # function

"""
Swish Activation function
"""
function swish(Z::Array{BigFloat})
    #Z₁ = sigmoid(Z).A
    #A = [Z[i] * Z₁[i] for i in 1:length(Z)]
    A = Z .* sigmoid(Z).A
    return (A = A, Z = Z)
end # function

"""
GELU Activation function
"""
function gelu(Z)
    A = 0.5 * Z * (1 + tanh( sqrt(pi / 2) * (Z + 0.044715 * Z^3) ))
    return (A = A, Z = Z)
end # function


#= Given a number n, print its prime factorization. We could do it
normal way which will be having time complexity of O (sqrt(N))
but we will be using Sieve to lessen the time complexity
We will make a sieve to get all the prime numbers and then get the
prime factorization of the number using that.=#

## Function

function PrimeFactor(n)
    a = zeros(Int64, n)
    for i = 2:n
        a[i] = i
    end
    for i = 2:n
        if (a[i] == i)
            for j = (i*i):i:n
                if (a[j] == j)
                    a[j] = i
                end
            end
        end
    end
    while (n != 1)
        println(a[n])
        n = div(n, a[n])
    end
end

## Input

n = readline()
n = parse(Int64, n)

#Calling the function

PrimeFactor(n)

#=
Sample Test case
Input:
    n = 495
Output:
    3 3 5 11

Time Complexity: O( log N )
=#


# Exercise 1

function factorial2(n)
    k=1
    for i ∈ 1:n
        k = k * i
    end
    return k
end

factorial2(3)

# Exercise 2

function binomial_rv(n,p)
    count = 0
    U = rand(n)
    for i ∈ 1:n
        if U[i] < p
            count += 1
        end
    end
    return count
end

n=60
p=0.7

binomial_rv(n,p)

# Exercise 3

n = 10000
count = 0

for i ∈ 1:n
    x, y = rand(2)
    r = ((x-0.5)^2 + (y-0.5)^2)^(0.5) # distance from center of circle
    if r < 0.5 # radius
        count+=1
    end
end

pi_estimate = (count/n)/0.5^2

println("Estimated pi = $pi_estimate")

# Exercise 4

function flipcoin(n)
    payoff = 0
    count = 0
    x = rand(n)
    for i ∈ eachindex(x)
        if x[i] <= 0.5
            count += 1
        else
            count = 0
        end
        print(count)
        if count >= 3
            payoff = 1
        end
    end
    return (count,payoff)
end

# parameters
n = 10
flipcoin(n)

# Exercise 5

# parameters
n = 200
α = 0.9

x = zeros(n)

for i ∈ 1:n
    x[i+1] = α * x[i] + randn()
end
plot(x)

# Exercise 6

αs = [0.0, 0.8, 0.98]
n = 200
p = plot() # naming a plot to add to

for α in αs
    x = zeros(n + 1)
    x[1] = 0.0
    for t in 1:n
        x[t+1] = α * x[t] + randn()
    end
    plot!(p, x, label = "alpha = $α") # add to plot p
end
p # display plot

# Exercise 7
function firstpassage(threshold; tmax=100, α = 1.0, σ =0.2)
    x = zeros(tmax)
    x[1] = 1.0
    x[tmax] = 0.0
    for i in 2:tmax
        x[i] = α * x[i-1] + σ * randn()
        if x[i] < threshold # checks threshold
            return i # leaves function, returning draw number
        end
    end
    return tmax # if here, reached maxdraws
end

# parameters
tmax = 200
σ = 0.2
αs = [0.8, 1.0, 1.2]
threshold = 0.0

# sample using firstpassage
nsamp = 100
averages = zeros(0)

for α ∈ αs
    draws = zeros(nsamp)
    for i ∈ 1:100
        draws[i] = firstpassage(threshold, tmax=tmax, α = α, σ = σ)
    end
    average = mean(draws)
    push!(averages,average)
#    histogram!(h, draws, labels = "alpha = $α", bins= 20) # add to histogram h
end
averages
# h

# Exercise 8a

function f_prime(f; x0 = 2)

    # intialise loop
    crit = 1
    iter = 1
    del = 0.1
    normdiff = Inf

    fd_initial = (f(x0+del)-f(x0))/del
    fd_old = fd_initial

    while crit > 1.0E-8 && iter < 1000

        del = del/2 # shrink change
        fd_new = (f(x0+del)-f(x0))/del #calculate derivative

        crit =  norm(fd_new - fd_old) #how large is the change in derivative?

        # continuation
        iter = iter + 1
        fd_old = fd_new
    end
    return fd_old
end

# test with polynomial function
f(x) = (x-1)^3
x0 = 2

derivative = f_prime(f,x0=2)

# Newton Raphson algorithm

function NewtonRaphson(f, f_prime; x0 = 2, tolerance =1.0E-8 , maxiter = 1000)

    #initialise loop
    iter = 1
    normdiff = Inf

    x_old = x0 # starting point
    f_old = f(x0)

    while normdiff > tolerance && iter < maxiter

        fd = f_prime(f, x0 = x_old) # get derivative

        step = f(x_old)/fd # compute step

        x_new = x_old - step # compute expression to iterate over

        normdiff = norm(x_new - x_old)# do we converge?

        # continuation
        iter = iter+1
        x_old = x_new
    end
    return (root = x_old, normdiff = normdiff, iter = iter )
end

# peform Newton Raphson algorithm
f(x) = (x-1)^3
x0 = 2

sol = NewtonRaphson(f,f_prime, x0 = x0, tolerance =1.0E-8 , maxiter = 1000 )

println("root = $(sol.root), and |f(x) - x| = $(sol.normdiff) in $(sol.iter) iterations")


function falseposition(f::Function, xl::Number, xu::Number, ea::Number, maxIter::Int64)
  # Estimate roots of functions using the false position method
  xr = NaN; # Initial estimate of the root
  fxu = f(xu);
  fxl = f(xl);

  for i = 1:maxIter
    xrold = xr;
    xr = xu - fxu*(xl-xu) / (fxl-fxu);
    fxr = f(xr);
    relativeError = abs((xr - xrold) / xr) * 100;

    if fxr * fxl == 0
      break;
    elseif fxr * fxl > 0
      xl = xr;
      fxl = fxr;
    else
      xu = xr;
      fxu = fxr;
    end

    if relativeError < ea
      break;
    end
  end
  return xr;
end


using Random, Distributions, NLsolve
Random.seed!(0)

a, b, c = 3, 5, 4
dist = TriangularDist(a,b,c)
n = 2000
samples = rand(dist,n)

m_k(k,data) = 1/n*sum(data.^k)
mHats = [m_k(i,samples) for i in 1:3]

function equations(F, x)
    F[1] = 1/3*( x[1] + x[2] + x[3] ) - mHats[1]
    F[2] = 1/6*( x[1]^2 + x[2]^2 + x[3]^2 + x[1]*x[2] + x[1]*x[3] +
		 x[2]*x[3] ) - mHats[2]
    F[3] = 1/10*( x[1]^3 + x[2]^3 + x[3]^3 + x[1]^2*x[2] + x[1]^2*x[3] +
		 x[2]^2*x[1] + x[2]^2*x[3] + x[3]^2*x[1] + x[3]^2*x[2] +
		 x[1]*x[2]*x[3] ) - mHats[3]
end

nlOutput = nlsolve(equations, [ 0.1; 0.1; 0.1])
sol = sort(nlOutput.zero)
aHat, bHat, cHat = sol[1], sol[3], sol[2]
println("Found estimates for (a,b,c) = ", (aHat, bHat, cHat) , "\n" )
println(nlOutput)


"""
    mean(nums)

Find mean of a vector of numbers

# Example

```julia
mean([3, 6, 9, 12, 15, 18, 21])      # returns 12.0
mean([5, 10, 15, 20, 25, 30, 35])    # returns 20.0
mean([1, 2, 3, 4, 5, 6, 7, 8])       # returns 4.5
```

Contributed By:- [Ashwani Rathee](https://github.com/ashwani-rathee)
"""
function mean(nums)
    return sum(nums) / length(nums)
end


# This file is a part of SimilaritySearch.jl

export L1Distance, L2Distance, SqL2Distance, LInftyDistance, LpDistance
import Distances: evaluate

"""
    L1Distance()

The manhattan distance or ``L_1`` is defined as

```math
L_1(u, v) = \\sum_i{|u_i - v_i|}
```
"""
struct L1Distance <: PreMetric end

"""
    L2Distance()

The euclidean distance or ``L_2`` is defined as

```math
L_2(u, v) = \\sqrt{\\sum_i{(u_i - v_i)^2}}
```
"""
struct L2Distance <: PreMetric end

"""
    SqL2Distance()

The squared euclidean distance is defined as

```math
L_2(u, v) = \\sum_i{(u_i - v_i)^2}
```

It avoids the computation of the square root and should be used
whenever you are able do it.
"""
struct SqL2Distance <: PreMetric end

"""
    LInftyDistance()

The Chebyshev or ``L_{\\infty}`` distance is defined as

```math
L_{\\infty}(u, v) = \\max_i{\\left| u_i - v_i \\right|}
```

"""
struct LInftyDistance <: PreMetric end

"""
    LpDistance(p)
    LpDistance(p, pinv)

The general Minkowski distance ``L_p`` distance is defined as

```math
L_p(u, v) = \\left|\\sum_i{(u_i - v_i)^p}\\right|^{1/p}
```

Where ``p_{inv} = 1/p``. Note that you can specify unrelated `p` and `pinv` if you need an specific behaviour.
"""
struct LpDistance <: PreMetric
    p::Float32
    pinv::Float32
end

LpDistance(p::Real) = LpDistance(p, 1/p)

"""
    evaluate(::L1Distance, a, b)

Computes the Manhattan's distance between `a` and `b`
"""
function evaluate(::L1Distance, a, b)
    d = zero(eltype(a))

    @fastmath @inbounds @simd for i in eachindex(a)
	    m = a[i] - b[i]
        d += ifelse(m > 0, m, -m)
    end

    d
end

"""
    evaluate(::L2Distance, a, b)

Computes the Euclidean's distance betweem `a` and `b`
"""
function evaluate(::L2Distance, a, b)
    d = zero(eltype(a))

    @fastmath @inbounds @simd for i in eachindex(a)
        d += (a[i] - b[i])^2 #m * m
    end

    sqrt(d)
end

"""
    evaluate(::SqL2Distance, a, b)

Computes the squared Euclidean's distance between `a` and `b`
"""
function evaluate(::SqL2Distance, a, b)
    d = zero(eltype(a))

    @fastmath @inbounds @simd for i in eachindex(a)
        d += (a[i] - b[i])^2
    end

    d
end

"""
    evaluate(::LInftyDistance, a, b)

Computes the maximum distance or Chebyshev's distance
"""
function evaluate(::LInftyDistance, a, b)
    d = zero(eltype(a))

    @inbounds @simd for i in eachindex(a)
        m = abs(a[i] - b[i])
        d = max(d, m)
    end

    d
end

"""
    evaluate(lp::LpDistance, a, b)

Computes generic Minkowski's distance
"""
function evaluate(lp::LpDistance, a, b)
    d = zero(eltype(a))

    @inbounds @simd for i in eachindex(a)
        m = abs(a[i] - b[i])
        d += m ^ lp.p
    end

    d ^ lp.pinv
end


using Plots
"""
    newton(f, df)
    newton(f, df, x0)
    newton(f, df, x0, tol)

Find roots of the noninear function `f` using Netwon's method.

# Argument
- `f::Function`: the nonlinear function.
- `df::function`: the derivative of the nonlinear function.
- `x0::Float64`: initial guess.
- 'tol::Float64': error tolerance.

# Example
```julia-repl
julia> newton(x->cos(x)-x, x->-sin(x)-1)
```
"""
function newton(f::Function, df::Function, x0::Float64=1.0, tol::Float64=1e-7)
    err = 1.0
    it = 0
    xp = x0
    while err > tol
        xc = x0 - f(xp)/df(xp)
        err = abs(xc-xp)
        xp = xc
        it += 1
    end
    return xp, it
end

"""
    secant(f)
    secant(f, x0, x1)
    secant(f, x0, x1, tol)

Find roots of the noninear function `f` using secant method.

# Argument
- `f::Function`: the nonlinear function.
- `x0::Float64`: the first initial guess.
- `x1::Float64`: the second initial guess.
- `tol::Float64`: error tolerance.

# Example
```julia-repl
julia> secant(x->cos(x)-x)
```
"""
function secant(f::Function, x0=1, x1 = 2, tol=1e-14)
    err = 1.0
    it = 0
    xp1 = x1
    xp2 = x0
    while err > tol
        xc = xp1 - f(xp1)*(xp1-xp2)/(f(xp1)-f(xp2))
        err = abs(xp1-xc)
        xp2 = xp1
        xp1 = xc
        it += 1
    end
    return xp1, it
end

"""
    bisect(f, a, b)
    bisect(f, a, b, tol)
    bisect(f, a, b, tol, maxit)

Find roots of the noninear function `f` using bisection.

# Argument
- `f::Function`: the nonlinear function.
- `a::Float64`: the lower bound of the interval.
- `b::Float64`: the upper bound of the interval.
- `tol::Float64`: error tolerance.
- `maxit::Int64`: maximum number of iteration.

# Example
```julia-repl
julia> bisection(x->cos(x)-x, 0, pi)
```
"""
function bisect(f::Function, a::Number, b::Number, tol=1e-7, maxit=100)
    if b < a
        error("the input of interval is not correct")
    end
    if f(a)*f(b) > 0
        error("You must give an input with difference sign")
    end
    it = 0
    while (b-a)>tol && it < maxit
        c = (a+b)/2
        if f(c) == 0
            return c
        elseif f(a)*f(c) < 0
            b = c
        else
            a = c
        end
        it += 1
    end
    return (a+b)/2, it
end

"""
    fixedpoint(g, x0)
    fixedpoint(g, x0, tol)
    fixedpoint(g, x0, tol, maxit)

Find roots of the noninear function `x=g(x)` using fixed point iteration.

# Argument
- `g::Function`: the fixed point function.
- `x0::Float64`: the initial guess.
- `tol::Float64`: error tolerance.
- `maxit::Int64`: maximum number of iteration.

# Example
```julia-repl
julia> fixedpoint(x->cos(x)-x, 0, pi)
```
"""
function fixedpoint(g::Function, x0, tol=1e-15, maxIt=50)
    it = 0
    x1 = x0 + 1
    while abs(x1-x0)>tol && it < maxIt
        x1 = x0
        x0 = g(x0)
        it += 1
    end
    return x0, it
end

"""
    cobweb(g, a, b, x0, N)

Generate the cobweb plot associated with the orbits x_n+1=g(x_n).

# Argument
- `g::Function`: the fixed point function.
- `a::Float64`: the lower bound of the interval.
- `b::Float64`: the upper bound of the interval.
- `x0::Float64`: the initial guess.
- `N::Int64`: the number of iteration.

# Example
```julia-repl
julia> cobweb(x->cos(x), 0, pi, 1, 30)
```
"""
function cobweb(g::Function,a,b,x0,N)
    #generate N linearly space values on (a,b)
    x = collect(range(a, b, length=N+1))
    y = g.(x);
    plot(x,y,linecolor=:black, legend = false)
    plot!(x,x,linecolor=:black, legend = false)
    x[1] = x0
    #@gif for i in 1:N
    anim = @animate  for i in 1:N
        x[i+1] = g(x[i])
        plot!([x[i]; x[i]], [x[i], x[i+1]], linecolor=:red,legend = false)
        plot!([x[i]; x[i+1]], [x[i+1], x[i+1]], linecolor=:red,legend = false)
    end
    return anim
end

"""
    viznewton(f, df, a, b, x0, N)

Visualize the Newton's method for the nonlinear equation f(x)=0.

# Argument
- `f::Function`: the nonlinear function.
- `df::function`: the derivative of the nonlinear function.
- `a::Float64`: the lower bound of the interval.
- `b::Float64`: the upper bound of the interval.
- `x0::Float64`: the initial guess.
- `N::Int64`: the number of iteration.

# Example
```julia-repl
julia> cobweb(x->cos(x), 0, pi, 1, 30)
```
"""
function viznewton(f::Function, df::Function, a,b,x0,N)
    #generate N linearly space values on (a,b)
    x = collect(range(a, b, length=N+1))
    y = f.(x);
    plot(x,y,linecolor=:black, legend = false)
    plot!(x,zeros(N+1),linecolor=:black, legend = false)
    x[1] = x0
    #@gif for i in 1:N
    anim = @animate  for i in 1:N
        x[i+1] = x[i] - f(x[i])/df(x[i])
        plot!([x[i]; x[i]], [0, f(x[i])], linecolor=:red,legend = false)
        plot!([x[i]; x[i+1]], [f(x[i]), 0], linecolor=:red,legend = false)
    end
    return anim
end

"""
    brent(f, a, b, tol, maxit)

Find roots of the noninear function `f` using Brent algorithm

# Argument
- `f::Function`: the nonlinear function.
- `a::Float64`: the lower bound of the interval.
- `b::Float64`: the upper bound of the interval.
- `tol::Float64`: error tolerance.
- `maxit::Int64`: maximum number of iteration.

# Example
```julia-repl
julia> brent(x->cos(x)-x, 0, pi)
```
"""
function brent(f::Function, a::Float64, b::Float64, tol::Float64=1e-13)
    if f(a)*f(b) >= 0.0
        error("Error: the root is not bracket")
    end
    if abs(f(a)) < abs(f(b))
        a, b = b, a
    end
    c = a
    d = c
    mflag = 1
    while abs(f(b)) > tol || abs(b-a) > tol
        if ~isapprox(f(a), f(c)) && ~isapprox(f(b), f(c))
            # inverse quadratic interpolation
            s = a*f(b)*f(c)/(f(a)-f(b))/(f(a)-f(c)) -
                b*f(a)*f(c)/(f(b)-f(a))/(f(b)-f(c)) -
                c*f(a)*f(b)/(f(c)-f(a))/(f(c)-f(b))
        else
            # secan method
            s = b - f(b)*(b-a)/(f(b)-f(a))
        end
        if (s > max((3a+b)/4, b) || s < min((3a+b)/4, b)) ||
            ( mflag == 1 && abs(s-b) >= abs(b-c)/2.0) ||
            ( mflag == 0 && abs(s-b) >= abs(c-d)/2.0) ||
            ( mflag == 1 && abs(b-c) < tol) ||
            ( mflag == 0 && abs(c-d) < tol)
            s = (a+b)/2
            mflag = 1
        else
            mflag = 0
        end
        d = c
        c = b
        if f(a)*f(s) < 0
            b = s
        else
            a = s
        end
        if abs(f(a)) < abs(f(b))
            a, b = b, a
        end
    end
    return b
end


using DifferentialEquations

f(u,t) = 0.98*u
u0 = 1.0
tspan = (0.0,5.0)
prob = ODEProblem(f,u0,tspan)
sol = solve(prob)

using Plots; gr()
plot(sol)

plot(sol,linewidth=5,title="Solution to the linear ODE with a thick line",
     xaxis="Time (t)",yaxis="u(t) (in μm)",label="My Thick Line!")

plot!(sol.t,t->0.5*exp(1.01t),lw=3,ls=:dash,label="True Solution!")

###############
# Solving system of Equaitons

function lorenz(du,u,p,t)
    du[1] = 10.0 * (u[2] - u[1])
    du[2] = u[1] * (28.0 - u[3] - u[2])
    du[3] = u[1] * u[2] - (8.0/3.0) * u[3]
end

u0 = [1.0;0.0;0.0]
tspan = (0.0,1.0)

prob = ODEProblem(lorenz,u0,tspan)
sol = solve(prob)
plot(sol , vars=[1,2,3])
# animate(sol,lw=3,every=4)
function lorenz(du,u,p,t)
 du[1] = 10.0*(u[2]-u[1])
 du[2] = u[1]*(28.0-u[3]) - u[2]
 du[3] = u[1]*u[2] - (8/3)*u[3]
end

u0 = [1.0;0.0;0.0]
tspan = (0.0,100.0)
prob = ODEProblem(lorenz,u0,tspan)
gr()
sol = solve(prob,vars=[1,2,3])

using StaticArrays

#######################
using DiffEqOperators

order = 2
deriv = 2

Δx =  0.1
N = 9
A = DerivativeOperator{Float64}(order,deriv,Δx,N,:Dirichlet0,:Dirichlet0)

full(DerivativeOperator{Float64}(order,deriv,Δx,N,:Dirichlet0,:Dirichlet0))

Missing around and learning DifferentialEquations.jl/Juliacon_2017_code.jl


module ActivationFunctions

export sigmoid, relu, softmax

"""
    sigmoid(x)

Computes the sigmoid activation of scalar `x`, returning a scalar.
"""
sigmoid(x::Number)::Number = (1.0 + exp(-x))^(-1)

"""
    sigmoid(x)

Computes the sigmoid activation of vector `x`, returning a vector.
"""
sigmoid(x::AbstractVector{<:Number})::AbstractVector{<:Number} = sigmoid.(x)

"""
    relu(x)

Computes the rectified linear unit activation of scalar `x`, returning a scalar.
"""
relu(x::Number)::Number = max(0.0, x)

"""
    relu(x)

Computes the rectified linear unit activation of vector `x`, returning a vector.
"""
relu(x::AbstractVector{<:Number})::AbstractVector{<:Number} = max.(0.0, x)

"""
    softmax(x)

Computes the softmax activation of vector `x`, returning a vector.

Unlike some other activation functions, softmax does not treat vector elements
independently.
"""
softmax(x::AbstractVector{<:Number})::AbstractVector{<:Number} =
    exp.(x .- maximum(x)) .* sum(exp.(x .- maximum(x)))^(-1)

end # module


# Starting in the top left corner of a 2×2 grid, and only being able to move to
# the right and down, there are exactly 6 routes to the bottom right corner.
#
# How many such routes are there through a 20×20 grid?

using ProjectEulerSolutions

# Memoization type solution to store intermediate results.
function p015solution(n::Integer=2)::Integer
    counts = zeros(Integer, n+1, n+1)
    counts[1, :] .= 1  # There's only one way down.
    counts[:, 1] .= 1  # There's only one way to the right.

    # Count for a cell is the sum of the counts above and to the left.
    for i = 2:n+1
        for j = 2:n+1
            @inbounds counts[i, j] = counts[i-1, j] + counts[i, j-1]
        end
    end

    return counts[end, end]
end

p015 = Problems.Problem(p015solution)

Problems.benchmark(p015, 20)


#using TensorToolbox, LinearAlgebra

#Define tensor as multidimensional arrays and calculate its norm:

X=rand(4,3,2)
norm(X)

#Create identity and diagonal tensor:

Id=neye(2,2,2)
D=diagt([1,2,3,4])

#For two tensors of same size calculate their inner product:

X=rand(3,3,3,3);Y=rand(3,3,3,3);
innerprod(X,Y)

#Matricization of a tensor:

X=rand(4,3,2);mode=1;
A=tenmat(X,mode)
B=tenmat(X,row=[2,1],col=3) #by row modes [2,1] and column mode 3

#Fold matrix back to tensor:

X=matten(A,mode,[4,3,2])
X=matten(B,[2,1],[3],[4,3,2]) # by row modes [2,1] and column mode 3

#n-mode product of a tensor and a matrix or an array of matrices:

X=rand(5,4,3);
A=[rand(2,5),rand(2,4),rand(2,3)];
ttm(X,A[1],1)  #X times A[1] by mode 1
ttm(X,[A[1],A[2]],[1,2]) #X times A[1] by mode 1 and times A[2] by mode 2; same as ttm(X,A,-3)
ttm(X,A) #X times matrices from A by each mode

#n-mode (vector) product of a tensor and a vector or an array of vectors:

X=rand(5,4,3);
V=[rand(5),rand(4),rand(3)];
ttv(X,V[1],1)  #X times V[1] by mode 1
ttv(X,[V[1],V[2]],[1,2]) #X times V[1] by mode 1 and times V[2] by mode 2; same as ttm(X,V,-3)
ttv(X,V) #X times vectors from V by each mode

#Outer product of two tensors:

 X=rand(5,4,3,2);Y=rand(2,3,4);
 ttt(X,Y)

#Kronecker product of two tensors (straightforward generalization of Kronecker product of matrices):

X=rand(5,4,3);Y=rand(2,2,2);
tkron(X,Y)

#The n-rank and the mutlilinear rank of a tensor:

X=rand(5,4,3);
mode=2;
nrank(X,mode)
mrank(X)

#The HOSVD:

X=rand(5,4,3);
hosvd(X) #same as T1=hosvd(X,eps_abs=1e-8)
hosvd(X,eps_abs=1e-6) #discard singular values lower than 1e-5
hosvd(X,eps_rel=1e-3) #discard singular values lower than 1e-3*sigma_{max}
hosvd(X,reqrank=[2,2,2])

#The CP decomposition:

X=rand(5,4,3);
R=3; #number of components
cp_als(X,R)  #same as cp_als(X,R,init="rand",dimorder=1:ndims(X))
cp_als(X,R,init=[rand(5,3),rand(4,3),rand(3,3)]) #initialize factor matrices
cp_als(X,R,init="nvecs",dimorder=[2,1,3])

##Tensors in Tucker format

#Define tensor in Tucker format by its core tensor and factor matrices:

F=rand(5,4,3);
A=[rand(6,5),rand(6,4),rand(6,3)];
ttensor(F,A)

#Get Tucker format of a tensor by using HOSVD:

X=rand(8,9,7);
hosvd(X)
hosvd(X,reqrank=[3,3,3]) #HOSVD with predefined multilinear rank

#Create random tensor in Tucker format of size 5x4x3 and mulilinear rank (2,2,2):

X=randttensor([5,4,3],[2,2,2])

#Basic functions:

size(X)
coresize(X)
ndims(X)
norm(X)
full(X)  #Creates full tensor out of Tucker format
reorth(X) #Orthogonalize factor matrices
permutedims(X,[2,1,3])

#n-mode matricization of a tensor in Tucker format:

mode=1;
tenmat(X,mode)

#Basic operations:

X=randttensor([5,4,3],[2,2,2]);Y=randttensor([5,4,3],[3,2,1]);
innerprod(X,Y)
X+Y
X-Y
X==Y #same as isequal(X,Y)
3*X #same as mtimes(3,X)

#n-mode product of a tensor in Tucker format and a matrix or an array of matrices:

X=randttensor([5,4,3],[2,2,2]);
A=[rand(2,5),rand(2,4),rand(2,3)];
ttm(X,A[1],1)  #X times A[1] by mode 1
ttm(X,[A[1],A[2]],[1,2]) #X times A[1] by mode 1 and times A[2] by mode 2; same as ttm(X,A,-3)
ttm(X,A) #X times matrices from A by each mode

#n-mode (vector) product of a tensor in Tucker format and a vector or an array of vectors:

X=randttensor([5,4,3],[2,2,2]);
V=[rand(5),rand(4),rand(3)];
ttv(X,V[1],1)  #X times V[1] by mode 1
ttv(X,[V[1],V[2]],[1,2]) #X times V[1] by mode 1 and times V[2] by mode 2; same as ttm(X,V,-3)
ttv(X,V) #X times vectors from V by each mode

#The n-rank and the mutlilinear rank of a tensor in Tucker format:

X=randttensor([9,8,7],[5,4,3]);
mode=2;
nrank(X,mode)
mrank(X)

#HOSVD of a tensor in Tucker format:

X=randttensor([6,7,5],[4,4,4]);
hosvd(X)  #same as hosvd(X,eps_abs=1e-8)
hosvd(X,eps_abs=1e-6) #discard singular values lower than 1e-5
hosvd(X,eps_rel=1e-3) #discard singular values lower than 1e-3*sigma_{max}
hosvd(X,reqrank=[3,3,3]) #HOSVD with predefined multilinear rank

#The CP decomposition:

X=randttensor([6,7,5],[4,4,4]);
R=3; #number of components
cp_als(X,R)  #same as cp_als(X,R,init="rand",dimorder=1:ndims(X))
cp_als(X,R,init=[rand(6,3),rand(7,3),rand(5,3)]) #initialize factor matrices
cp_als(X,R,init="nvecs",dimorder=[2,1,3])

##Tensors in Kruskal format

#Define tensor in Kruskal format by its factor matrices (and vector of weights):

lambda=rand(3)
A=[rand(5,3),rand(4,3),rand(3,3)];
ktensor(A)
ktensor(lambda,A)

#Create random tensor in Kruskal format of size 5x4x3 and with 2 components:

X=randktensor([5,4,3],2)

#Basic functions:

size(X)
ndims(X)
norm(X)
full(X)  #Creates full tensor out of Kruskal format
permutedims(X,[2,1,3])
ncomponents(X) #Number of components

#n-mode matricization of a tensor in Kruskal format:

mode=1;
tenmat(X,mode)

#Basic operations:

X=randktensor([5,4,3],2);Y=randktensor([5,4,3],3);
innerprod(X,Y)
X+Y
X-Y
X==Y #same as isequal(X,Y)
3*X #same as mtimes(3,X)

#n-mode product of a tensor in Kruskal format and a matrix or an array of matrices:

X=randktensor([5,4,3],2);
A=[rand(2,5),rand(2,4),rand(2,3)];
ttm(X,A[1],1)  #X times A[1] by mode 1
ttm(X,[A[1],A[2]],[1,2]) #X times A[1] by mode 1 and times A[2] by mode 2; same as ttm(X,A,-3)
ttm(X,A) #X times matrices from A by each mode

#n-mode (vector) product of a tensor in Kruskal format and a vector or an array of vectors:

X=randktensor([5,4,3],2);
V=[rand(5),rand(4),rand(3)];
ttv(X,V[1],1)  #X times V[1] by mode 1
ttv(X,[V[1],V[2]],[1,2]) #X times V[1] by mode 1 and times V[2] by mode 2; same as ttm(X,V,-3)
ttv(X,V) #X times vectors from V by each mode

#Arrange the rank-1 components of a tensor in Kruskal format:

X=randktensor([6,5,4,3],3);
arrange(X)
arrange!(X)

#Fix sign ambiguity of a tensor in Kruskal format:

X=randktensor([6,5,4,3,4],3);
fixsigns(X)
fixsigns!(X)

#Distribute weights a tensor in Kruskal format to a specific mode:

X=randktensor([3,3,3],3);
mode=2;
redistribute(X,mode)
redistribute!(X,mode)

#The CP decomposition:

X=randktensor([6,7,5],4);
R=3; #number of components
cp_als(X,R)  #same as cp_als(X,R,init="rand",dimorder=1:ndims(X))
cp_als(X,R,init=[rand(6,3),rand(7,3),rand(5,3)]) #initialize factor matrices
cp_als(X,R,init="nvecs",dimorder=[2,1,3]);

##Tensors in Hierarchical Tucker format

#Define tensor in Hierarchical Tucker format by dimensional tree T, its transfer tensors and factor matrices:

T=dimtree(3)
B=[rand(2,3,1),rand(4,3,2)]
A=[rand(5,4),rand(4,3),rand(3,3)]
htensor(T,B,A)

#Get Tucker format of a tensor by using htrunc:

X=rand(8,9,7);
htrunc(X)
htrunc(X,maxrank=3) #hrunc with defined maximal rank

#Create random tensor in Hierarchical Tucker format of size 5x4x3:

X=randhtensor([5,4,3])

#Basic functions:

size(X)
ndims(X)
norm(X)
full(X)  #Creates full tensor out of Hierarchial Tucker format
reorth(X) #Orthogonalize factor matrices

#Basic operations:

X=randhtensor([5,4,3]);Y=randhtensor([5,4,3]);
innerprod(X,Y)
X+Y
X-Y
X==Y #same as isequal(X,Y)
3*X #same as mtimes(3,X)

#n-mode product of a tensor in Hierarchical Tucker format and a matrix or an array of matrices:

X=randhtensor([5,4,3]);
A=[rand(2,5),rand(2,4),rand(2,3)];
ttm(X,A[1],1)  #X times A[1] by mode 1
ttm(X,[A[1],A[2]],[1,2]) #X times A[1] by mode 1 and times A[2] by mode 2; same as ttm(X,A,-3)
ttm(X,A) #X times matrices from A by each mode

#n-mode (vector) product of a tensor in Hierarchical Tucker format and a vector or an array of vectors:

X=randhtensor([5,4,3]);
V=[rand(5),rand(4),rand(3)];
ttv(X,V[1],1)  #X times V[1] by mode 1
ttv(X,[V[1],V[2]],[1,2]) #X times V[1] by mode 1 and times V[2] by mode 2; same as ttm(X,V,-3)
ttv(X,V) #X times vectors from V by each mode

#The h-rank of a tensor in Hierarchical Tucker format:

X=htrunc(rand(9,8,7),maxrank=2)
hrank(X)

##Tensors in Tensor Train format

#Define tensor in TT format by its core tensors:

G=CoreCell(undef,3)
G[1]=rand(1,4,3)
G[2]=rand(3,6,4)
G[3]=rand(4,3,1)
X=TTtensor(G)

#Get TT format of a tensor by using TTsvd:

X=rand(5,4,3,2)
TTsvd(X)
TTsvd(X,reqrank=[2,2,2])

#Create random TT tensor of size 5x4x3 and TT-rank (2,2):

X=randTTtensor([5,4,3],[2,2])

#Basic functions::

size(X)
TTrank(X)
ndims(X)
norm(X)
full(X)  #Creates full tensor out of Tucker format
reorth(X)

#Basic operations:

X=randTTtensor([5,4,3],[2,2])
Y=randTTtensor([5,4,3],[3,3])

innerprod(X,Y)
X+Y
X-Y
3*X

#TTsvd of a TT tensor:

X=randTTtensor([7,6,5],[5,4])
TTsvd(X,reqrank=[3,3])

println()


# # Exercise: compressing an image using the SVD

using Pkg
Pkg.add("Images")
Pkg.add("ImageMagick")

using Images, LinearAlgebra

banana = load("Courses/Introduction to Julia/images/banana.jpg")

# Images work just like arrays, but display specially

size(banana)

banana[60,50]

gb = Gray.(banana)

channelview(gb)

# The SVD factorizes a matrix A such that
#     A == U * S * transpose(V)
# where U and V are unitary, and S is diagonal.
#
# You can think of this as a sum of outer products (column of U times row
# of V), where each one is scaled by a value in diag(S).
# Each outer product is a full matrix (image).
# So, the ones scaled by larger singular values are more important,
# and we can throw away ones scaled by smaller values.

# Here's what it looks like keeping 30, 10, 5, and 3 values:

load("Courses/Introduction to Julia/images/banana_30svals.png")

load("Courses/Introduction to Julia/images/banana_10svals.png")

load("Courses/Introduction to Julia/images/banana_5svals.png")

load("Courses/Introduction to Julia/images/banana_3svals.png")

# ## Problem statement
#
# Write a function `compress_image`. Its arguments should be an image and the
# factor by which you want to compress the image. A compressed grayscale image
# should display when compress_image is called.
# For example,
#     `compress_image("images/104_100.jpg", 33)`
# will return a compressed image of a grayscale banana built using 3 singular
# values. (This image has 100 singular values, so use fld(100, 33) to determine
# how many singular values to keep. fld performs "floor" division.)
#
# Hints:
# - Perform the SVD on the `channelview` of the image.
# - Read the documentation for `svd`.

# Remember: you can index arrays with `:` and ranges `a:b`:

A = [ i+j for i = 1:10, j = 1:10 ]

A[2:4, :]
A[1:2, 1:2]
A[:, 4:5]

Courses/Introduction to Julia/05. Image compression exercise.jl


using LinearAlgebra
"""
    lu!(A)
Compute LU factorization of the matrix `A`.

# Example
```julia-repl
julia> lu!([1 2; 3 4])
```
"""
function lu!(A::AbstractMatrix)
    n,n = size(A)
    p = collect(1:n)
    for k in 1:n-1
        m = argmax(abs.(A[k:n,k]))
        m = m + k - 1
        if A[m,k] != 0
            if m != k
                p[k], p[m] = p[m], p[k]
                for j in 1:n
                    A[k,j], A[m,j] = A[m,j], A[k,j]
                end
            end
            for i in k+1:n
                A[i,k] /= A[k,k]
                for j in k+1:n
                    A[i,j] -= A[i,k]*A[k,j]
                end
            end
        end
    end
    L = tril(A,-1) + Matrix{Float64}(I, n, n)
    U = triu(A)
    return L, U, p
end

"""
    chol!(A)
Compute cholesky factorization of the spd matrix `A`.

# Argument
- `A:AbstractMatrix`: symmetric positive definite matrix

# Example
```julia-repl
julia> chol([1 2; 2 5])
```
"""
function chol(A::AbstractMatrix)
    n,n = size(A)
    L = similar(A)
    @inbounds begin
         for i in 1:n
             tmp = 0
             for k in 1:i-1
                tmp += L[i,k].^2
            end
            L[i,i] = sqrt(A[i,i] - tmp)
            for j in i+1:n
                L[j,i] = A[i,j] / L[i,i]
                for k in 1:i-1
                    L[j,i] -=  L[i,k].*L[j,k] / L[i,i]
                end
            end
        end
    end
    return L
end

"""
    gepp!(A,b)
Solve the linear system ``Ax=b`` using Gaussian elimination with partial pivot

# Example
```julia-repl
julia> A = rand(3,3)
julia> b = rand(3)
julia> x = gepp!(A, b)
```
"""
function gepp!(A::AbstractMatrix, b::AbstractVector)
    n,n = size(A)
    for k in 1:n-1
        # pivot
        m = argmax(abs.(A[k:n,k]))
        m = m + k - 1
        if A[m,k] != 0
            if m != k
                b[k], b[m] = b[m], b[k]
                for j in 1:n
                    A[k,j], A[m,j] = A[m,j], A[k,j]
                end
            end
            for i in k+1:n
                A[i,k] /= A[k,k]
                b[i] -= A[i,k]*b[k]
                for j in k+1:n
                    A[i,j] -= A[i,k]*A[k,j]
                end
            end
        end
    end
    # back substituion
    x = zeros(n)
    for k in n:-1:1
        for j = n:-1:k+1
            b[k] -= A[k,j]*x[j]
        end
        x[k] = b[k] / A[k,k]
    end
    return x
end

"""
    jacobi(A, b, x0)
    jacobi(A, b, x0, tol)

Solve the linear system using Jacobi method.

# Example
```julia-repl
julia> A = rand(3,3)
julia> b = rand(3)
julia> x0 = rand(3)
julia> x = jacobi(A, b, x0)
```
"""
function jacobi(A::AbstractMatrix, b::AbstractVector, x::AbstractVector, tol::Float64=1e-10)
    xc = similar(b)
    n, = size(b)
    err = 1.0
    it = 0
    while err > tol
        @inbounds for i in 1:n
            xc[i] = b[i]
            @inbounds for k in 1:n
                if k != i
                    xc[i] -= A[i,k]*x[k]
                end
            end
            xc[i] /= A[i,i]
        end
        err = norm(x-xc)
        x = copy(xc)
        it += 1
    end
    return xc, it
end

"""
    gaussseidel(A, b, x0)
    gaussseidel(A, b, x0, tol)

Solve the linear system using Gauss-Seidel method.

# Example
```julia-repl
julia> A = rand(3,3)
julia> b = rand(3)
julia> x0 = rand(3)
julia> x = gaussseidel(A, b, x0)
```
"""
function gaussseidel(A::AbstractMatrix, b::AbstractVector, x::AbstractVector, tol::Float64=1e-10)
    n, = size(b)
    err = 1.0
    it = 0
    xc = copy(x)
    while err > tol
        @inbounds for i in 1:n
            xc[i] = b[i]
            @inbounds for k in 1:n
                if k != i
                    xc[i] -= A[i,k]*xc[k]
                end
            end
            xc[i] /= A[i,i]
        end
        err = norm(x-xc)
        x = copy(xc)
        it += 1
    end
    return xc, it
end


using Distributions, Random
Random.seed!(1234)

# Sample from conditional X|Y=y
# Inputs:
# - y: The current value of y (Y_k)
# - μ: The mean vector of the joint density
# - Σ: The covariance matrix of X,Y
# Outputs:
# X_{k+1} where X_{k+1} ∼ f(X|Y=y)
function condition_on_Y(y, μ, Σ)
    μX = μ[1]
    μY = μ[2]
    σX2, σXY, _, σY2 = Σ

    conditional_mean = μX + σXY/σY2*(y-μY)
    conditional_variance = (1-σXY^2/(σX2*σY2))*σX2

    distribution = Normal(conditional_mean, conditional_variance)

    return rand(distribution)
end

# Sample from conditional Y|X=y
# Inputs:
# - x: The current value of x (X_{k+1})
# - μ: The mean vector of the joint density
# - Σ: The covariance matrix of X,Y
# Outputs:
# Y_{k+1} where Y_{k+1} ∼ f(y|X=x)
function condition_on_X(x, μ, Σ)
    μX = μ[1]
    μY = μ[2]
    σX2, σXY, _, σY2 = Σ

    conditional_mean = μY + σXY/σX2*(x-μX)
    conditional_variance = (1-σXY^2/(σX2*σY2))*σY2

    distribution = Normal(conditional_mean, conditional_variance)

    return rand(distribution)
end

# Gibbs sampler to obtain samples from (X,Y) ~ N(μ,Σ)
# Inputs:
# - μ: The joint density mean vector
# - Σ: The covaraince matrix of X and Y
# - n: The number of samples to return
# Outputs:
# An (n x 2) matrix where each row represent a sample
function bivariate_gibbs(μ, Σ, n)
    # Store samples here
    S = zeros(n,2)

    # Initialize S_0 to means
    S[1,:] = μ

    for i=2:n
        X_k1 = condition_on_Y(S[i-1,2], μ, Σ) # Get X_{k+1}
        Y_k1 = condition_on_X(X_k1, μ, Σ) # Get Y_{k+1}
        S[i,:] = [X_k1, Y_k1]
    end

    return S
end

# Define joint density parameters
μX = -1
μY = 1
σX = √2
σY = √3
σXY = 2
μ = [μX, μY]
Σ = [σX^2 σXY; σXY σY^2]

# Get samples
samples = bivariate_gibbs(μ, Σ, 10000)

# Create 2d histogram
using Seaborn, Pandas
pygui(true)
df = DataFrame(Dict(:X=>samples[:,1], :Y=>samples[:,2]))
jointplot(x = "X", y = "Y", data = df, s = 1, alpha = 0.18)