@doc raw"""
    general_form(xi::Point2D, xo::Point2D)

Returns the general form equation of a line that passes through points `xi` and `xo` as a
[`SVector`](@ref) which holds A, B and C such that:
```math
A \cdot x + B \cdot y +  C = 0.
```
"""
function general_form(xi::Point2D, xo::Point2D)
    A = xi[2] - xo[2]
    B = xo[1] - xi[1]
    C = xi[1] * xo[2] - xo[1] * xi[2]
    ABC = SVector(A, B, C)
    ABC /= norm(ABC) # for numerical reasons (?)
    return ABC
end
# the function above is faster than using linear algebra:
# [x1 y1; x2 y2] ⋅ [Â, B̂] = [-1, -1] yields to solution for Â and B̂ such that
# Â ⋅ x + B̂ ⋅ y + 1 = 0. Here we make it faster.
# F = transpose(hcat(xi, xo))
# b = SVector(-1, -1)
# x = F \ b
# ABC = vcat(b, 1) # and then we can normalize

# We could use LazySets.jl for some part of this particular function but it seems slow!
"""
    intersections(mesh::Mesh, cell_id::Int, track::Track)

Computes the entry and exit points of a given `track` that is known to cross element with id
`cell_id`.
"""
function intersections(
    mesh::Mesh, cell_id::Int32, track::Track{BIn,BOut,DFwd,DBwd,T}
) where {BIn,BOut,DFwd,DBwd,T}
    @unpack model, cell_nodes = mesh

    node_coordinates = get_node_coordinates(get_grid(model))
    node_ids = cell_nodes[cell_id]

    int_points = MVector{4,Point2D{T}}(
        Point2D(0, 0), Point2D(0, 0), Point2D(0, 0), Point2D(0, 0)
    )

    n_int = 0
    parallel_found = false
    for i in eachindex(node_ids)

        j = i == lastindex(node_ids) ? 1 : i + 1

        # get node coordinates and cast them to Point2D
        p1 = convert(Point2D{T}, node_coordinates[node_ids[i]])
        p2 = convert(Point2D{T}, node_coordinates[node_ids[j]])

        # compute general form equation for the selected element face
        ABC = general_form(p1, p2)

        # compute intersections between track and element face
        are_parallel, x_int = intersection(track.ABC, ABC)

        # if the track is parallel to the element face, it can be on top of it or do not
        # cross at all. Either way, we just ignore this case because we can compute the
        # intersections using the other faces.
        if are_parallel
            parallel_found = true
            continue

        # if the intersection is outside the face, avoid it
        elseif !point_in_segment(p1, p2, x_int)
            continue

        # this is a valid intersection, store it
        else
            n_int += 1
            int_points[n_int] = x_int
        end
    end

    # now let's check all the possible situations and handle extreme cases
    if n_int in (3, 4)
        ℓ = zero(T)
        # get the points that have the maximun distance between
        for i in 2:n_int, j in i:n_int
            x1 = int_points[i-1]
            x2 = int_points[j]
            ℓi = norm(x1 - x2)
            if ℓi > ℓ
                x_int1 = x1
                x_int2 = x2
                ℓ = ℓi
            end
        end
        return order_intersection_points(track, x_int1, x_int2)

    elseif n_int == 2 && parallel_found

        x_int1 = int_points[1]
        x_int2 = int_points[2]
        return order_intersection_points(track, x_int1, x_int2)

    elseif n_int == 2 && !parallel_found

        x_int1 = int_points[1]
        x_int2 = int_points[2]

        if isapprox(x_int1, x_int2)
            # do nothing and move a tiny step forward in the parent function, we are on a
            # vertex
            return x_int1, x_int2
        else
            return order_intersection_points(track, x_int1, x_int2)
        end
    elseif iszero(n_int) || isone(n_int)
        # the parent function will move a tiny step further
        return Point2D{T}(0, 0), Point2D{T}(0, 0)
        # error("This is an unexpected case. Please, submit an issue.")
    end
end

"""
    intersection(ABC1, ABC2)

Computes the intersection between two lines with general form equations given by `ABC1` and
`ABC2`. It also indicates if the lines are parallel.
"""
function intersection(ABC1::AbstractVector, ABC2::AbstractVector)
    a = ABC1[2] * ABC2[1]
    b = ABC2[2] * ABC1[1]
    x = y = zero(a)
    are_parallel = isapprox(a, b)
    if !are_parallel
        det = a - b
        x = (ABC1[3] * ABC2[2] - ABC2[3] * ABC1[2]) / det
        y = (ABC1[1] * ABC2[3] - ABC2[1] * ABC1[3]) / det
    end
    return are_parallel, Point2D(x, y)
end

# this one returns a matrix even if F is not invertible (have the same speed as the above)
# function intersection(ABC1, ABC2)
#     F = @SMatrix [ABC1[1] ABC1[2]
#                   ABC2[1] ABC2[2]]
#     b = @SVector [ABC1[3], ABC2[3]]

#     x = F \ (-b)

#     return Point2D(x[1], x[2])
# end

function order_intersection_points(track::Track, x1::Point2D, x2::Point2D)
    @unpack ϕ = track

    if isless(ϕ, π/2)
        if x1[1] < x2[1]
            xi = x1
            xo = x2
        else
            xi = x2
            xo = x1
        end
    else
        if x1[1] > x2[1]
            xi = x1
            xo = x2
        else
            xi = x2
            xo = x1
        end
    end
    return xi, xo
end


#=
Compute the running median of a sequence of numbers. That is, given a stream of numbers, print out the median of the list so far on each new element.

Recall that the median of an even-numbered list is the average of the two middle numbers.

For example, given the sequence [2, 1, 5, 7, 2, 0, 5], your algorithm should print out:

2
1.5
2
3.5
2
2
2
=#

function print_running_median(arr::Array; store = false)
    if store
        median_arr = []
    end
    for index in 1:length(arr)
        current_arr = sort(arr[1:index])
        if length(current_arr) % 2 == 0
            median = (current_arr[Int8(length(current_arr)/2)] + current_arr[Int8(length(current_arr)/2 + 1)]) / 2
            println(median)
            if store
                push!(median_arr, median)
            end
        else
            median = current_arr[ceil(Int8, length(current_arr)/2)]
            println(median)
            if store
                push!(median_arr, median)
            end
        end
    end
    if store
        return median_arr
    end
end


using Statistics

"""
    mse(y, t; back = false)
Return the mean square error:
    mean((y .- t) .^ 2)
# Example
```
julia> loss = mse
mse (generic function with 1 method)

julia> y, t = [1, 3, 2, 5], [1, 2, 3, 4]
([1, 3, 2, 5], [1, 2, 3, 4])

julia> loss(y, t)
0.75
```
"""
function mse(y, t; back = false)
    if back == true
        return 2(y .- t)
    end
    return mean((y .- t) .^ 2)
end


using Plots
import Distributions
using Random
using Turing
using Statistics
using StatsPlots
using FillArrays

dataerr = 1.0
d = Cauchy(0,dataerr)

x = 0:10
y = 10.0 .* exp.(-x*0.1) .+ rand(d,length(x))

scatter(x,y,label=false)
#plot!(x_fit,y_fit,label=false)

@model function linear_fit(x,y)
    σ ~ Uniform(0.0,20.0)
    b ~ Uniform(-100.0,100.0)
    m ~ Uniform(-100.0,100.0)
    for i in 1:length(x)
        y[i] ~ Cauchy(b+m*x[i],σ)
    end
end

model_fit = linear_fit(x,y)
chain = sample(model_fit,NUTS(0.65),1000)
plot(chain)

mean(vec(chain[:b]))
std(vec(chain[:b]))

mean(vec(chain[:m]))
std(vec(chain[:m]))

@model function nonlinear_fit(x,y,f)
    σ ~ Uniform(0.0,20.0)
    b ~ Uniform(0.0,100.0)
    m ~ Uniform(0.0,1.0)
    for i in 1:length(x)
        y[i] ~ Cauchy(f(x[i],b,m),σ)
    end
end

function f(x,b,m)
    b*exp(-x*m)
end

model_nonlinear = nonlinear_fit(x,y,f)
chain2 = sample(model_nonlinear,NUTS(0.65),1000)

plot(chain2)


#Find Volumes of Various Shapes.
#Wikipedia reference: https://en.wikipedia.org/wiki/Volume
# contributed By:- [Qingyu Qu](https://github.com/ErikQQY)

"""
    vol_cube()

Compute the volume of a cube.

# Examples

```julia-repl
julia> vol_cube(1)
1
julia> vol_cube(3)
27
julia> vol_cube(-1)
DomainError
```
"""
function vol_cube(side)
    if side<0
        throw(DomainError("vol_cube() only works for non negative values."))
    end
    return side^3
end

"""
    vol_cuboid(width, height, length)

Compute the volume of a vol_cuboid

# Examples

```julia-repl
julia> vol_cuboid(1, 1, 1)
1
julia> vol_cuboid(1, 2, 3)
6
```
"""
function vol_cuboid(width, height, length)
    if width < 0 || height <0 || length < 0
        throw(DomainError("vol_cuboid() only works for non negative values."))
    end
    return width*height*length
end

"""
    vol_cone(area_of_base, height)

Compute the Volume of a Cone

# Examples

```julia-repl
julia> vol_cone(10, 3)
10.0
julia> vol_cone(1, 1)
0.3333333333333333
```
"""
function vol_cone(area_of_base, height)
    if area_of_base < 0 || height <0
        throw(DomainError("vol_cone() only works for non negative values."))
    end
    return area_of_base*height/3.0
end

"""
    vol_right_circ_cone(radius, height)

Compute the Volume of a Right Circular Cone.

# Examples

```julia-repl
julia> vol_right_circ_cone(2, 3)
12.566370614359172
```
"""
function vol_right_circ_cone(radius, height)
    if radius < 0 || height < 0
        throw(DomainError("vol_right_circ_cone() only works for non negative values."))
    end
    return π*radius^2*height/3.0
end

"""
    vol_prism(area_of_base, height)

Compute the Volume of a Prism.

# Examples

```julia-repl
julia> vol_prism(10, 2)
20.0
julia> vol_prism(11, 1)
11.0
```
"""
function vol_prism(area_of_base, height)
    if area_of_base < 0 || height < 0
        throw(DomainError("vol_prism() only works for non negative values."))
    end
    return area_of_base*height
end

"""
    vol_pyramid(area_of_base, height)

Compute the volume of a Pyramid.

# Examples

```julia-repl
julia> vol_pyramid(10, 3)
10.0
julia> vol_pyramid(1.5, 3)
1.5
```
"""
function vol_pyramid(area_of_base, height)
    if area_of_base < 0 || height < 0
        throw(DomainError("vol_pyramid() only works for non negative values."))
    end
    return area_of_base*height/3.0
end

"""
    vol_sphere(radius)

Compute the volume of a sphere.

# Examples

```julia
vol_sphere(5)  # returns 523.5987755982989
vol_sphere(1)  # returns 4.1887902047863905
vol_sphere(-1) # returns DomainError
```
"""
function vol_sphere(radius)
    if radius < 0
        throw(DomainError("vol_sphere() only works for non negative values."))
    end
    return 4/3*pi*(radius)^3
end

"""
    vol_circular_cylinder(area_of_, height)

Compute the Volume of a Circular Cylinder.

# Examples

```julia-repl
julia> vol_circular_cylinder(1, 1)
3.141592653589793
julia> vol_circular_cylinder(4, 3)
150.79644737231007
"""
function vol_circular_cylinder(radius, height)
    if radius < 0 || height < 0
        throw(DomainError("vol_circular_cylinder() only works for non negative values."))
    end
    return pi*radius^2*height
end


import FractionalCalculus.FracIntAlg

"""
Riemann-Liouville sense fractional integral algorithms

Note this two algorithms belong to direct compute, precise are ensured, but maybe cause more memory allocation and take more compilation time.
"""
abstract type RLInt <: FracIntAlg end

"""
# Riemann Liouville sense fractional integral

    fracint(f::Function, α, start_point, end_point, h, RL_Direct())

Riemann_Liouville fractional integral using complex step differentiation.

```math
(J^αf)(t)=\\frac{1}{\\Gamma(α)} \\int_0^t(t-τ)^{α-1}f(τ)dτ
```

By using [QuadGK](https://github.com/JuliaMath/QuadGK.jl) calculate the integration and obtain the value.

### Example:

```julia-repl
julia> fracint(x->x^5, 0.5, 0, 2.5, 1e-8, RL_Direct())
```

Returns a tuple (1.1639316474512205, 1.0183453796725215e-8), which contains the value of this derivative is 1.1639316474512205, and the error estimate is 1.0183453796725215e-8
"""
struct RL_Direct <: RLInt end

"""
# Riemann Liouville sense fractional integral with first diff known.

    fracint(f, fd, α, start_point, end_point, RL_Direct_First_Diff_Known())

### Example

```julia-repl
julia> fracint(x->x^5, x->5*x^4, 0.5, 0, 2.5, RL_Direct_First_Diff_Known())
```

With first derivative known, we can use the Riemann Liouville sense to obtain the fractional integral more effcient.
"""
struct RL_Direct_First_Diff_Known <: RLInt end

"""
@article{LI20113352,
title = {Numerical approaches to fractional calculus and fractional ordinary differential equation},
author = {Changpin Li and An Chen and Junjie Ye},
}
"""

"""
# Riemann Liouville sense fractional integral using piecewise interpolation.

    fracint(f, α, end_point, h, RL_Piecewise())

### Example

```julia-repl
julia> fracint(x->x^5, 0.5, 2.5, 0.0001, RL_Piecewise())
```

By deploying Piecewise interpolation to approximate the original function, with small step size, this method is fast and take little memory allocation.

Using piecewise linear interpolation:

```math
    y_n(t)=\\frac{t-t_{i+1}}{t_i-t_{i+1}}y(t_i)+\\frac{t-t_i}{t_{i+1}-t_i}y(t_{i+1})
```

Constitute the original function with the interpolation and implement the summation.
"""
struct RL_Piecewise <: RLInt end

"""
@book{oldham_spanier_1984,
title={The fractional calculus: Theory and applications of differentiation and integration to arbitrary order},
author={Oldham, Keith B. and Spanier, Jerome},
year={1984}}
"""

"""
# Riemann Liouville sense fractional integral approximation.

    fracint(f, α, end_point, h, RLInt_Approx())

### Example

```julia-repl
julia> fracint(x->x^5, 0.5, 2.5, 0.0001, RLInt_Approx())
```

Using the **Staircase approximation** to approximate the original function and implement the summation.
"""
struct RLInt_Approx <: RLInt end

"""
# Riemann Liouville sense fractional integral using **Linear interpolation**.

    fracint(f, α, end_point, h, RL_LinearInterp())

### Example

```julia-repl
julia> fracint(x->x^5, 0.5, 2.5, 0.0001, RL_LinearInterp())
```

**RL_LinearInterp** is more complex compared with *RLInt_Approx* but more precise.

Deploying the [Linear Interpolation](https://en.wikipedia.org/wiki/Linear_interpolation) between ``f_{j+1}`` and ``f_j``, **RL_LinearInterp** method is more precise than **RLInt_Approx**.
"""
struct RL_LinearInterp <: RLInt end

"""
@article{2009,
title={Matrix approach to discrete fractional calculus II: Partial fractional differential equations},
DOI={10.1016/j.jcp.2009.01.014},
author={Podlubny, Igor and Chechkin, Aleksei and Skovranek, Tomas and Chen, YangQuan and Vinagre Jara, Blas M.},
}

"""

"""
# Riemann Liouville sense integral using Triangular Strip Matrix to discrete.

    fracint(f, α, end_point, h, RLInt_Matrix())

Using Triangular Strip Matrix to approximate fractional integral.

### Example

```julia-repl
julia> fracint(x->x^5, 0.5, 2.5, 0.0001, RLInt_Matrix())
```

!!! info
    Triangular Strip Matrix method returns the derivative in the interval ``[0, T]`` in ```Vector```

!!! tip
    With the advancing Triangular Strip Matrix method, you can not only compute fractional integrals, integer order, higher order integral is also supported!!
Try to set α as an integer, arbitrary integer of course! I promise you would enjoy it😏

Using [Triangular Strip Matrix](https://en.wikipedia.org/wiki/Triangle_strip) to discrete the integral.
"""
struct RLInt_Matrix <: RLInt end

"""
# Riemann Liouville sense fractional integral using fractional Simpson's formula to approximate.

### Example

```julia-repl
julia> fracint(x->x, 0.5, 1, 0.001, RLInt_Simpson)
```

@inproceedings{Li2015NumericalMF,
  title={Numerical Methods for Fractional Calculus},
  author={Changpin Li and Fanhai Zeng},
  year={2015}
}

Using fractional Simpson's formula to discrete fractional integral.
"""
struct RLInt_Simpson <: RLInt end

"""
# Riemann Liouville sense fractional integral using fractional trapezoidal formula to approximate.

### Example

```julia-repl
julia> fracint(x->x, 0.5, 1, 0.001, RLInt_Trapezoidal())
```

《Numerical methods for fractional calculus》.
Using Trapezoidal method to approximate fractional integral
"""
struct RLInt_Trapezoidal <: RLInt end

"""
# Riemann Liouville sense fractional integral using fractional rectangular formula to approximate.

### Example

```julia-repl
julia> fracint(x->x, 0.5, 1, 0.001, RLInt_Rectangular())
```
"""
struct RLInt_Rectangular <: RLInt end

"""
# Riemann Liouville sense fractional integral using cubic spline interpolation to approximate.

### Example

```julia-repl
julia> fracint(x->x, 0.5, 1, 0.0001, RLInt_Cubic_Spline_Interp())
```

Error estimate of this method is ``\\mathcal{O(h^4)}``, it is determined by the error of the cubic spline interpolation.

!!! warning "Set h as 0.001 or bigger"
    For some reason, in the **RLInt_Cubic_Spline_Interp** method, set **h** as 0.001 would get better result.

> Numerical Methods for Fractional Calculus Page 34
"""
struct RLInt_Cubic_Spline_Interp <: RLInt end

####################################
###     Type defination done     ###
####################################

"""
Check if the format of nargins is correct
"""
function checks(f, α, start_point, end_point)
    α % 1 != 0 ? nothing : error("Decimal number only!")
    if isa(end_point, Number)
        start_point < end_point ? nothing : DomainError("Domain error! Start point must smaller than end point")
    elseif isa(end_point, AbstractArray)
        start_point < minimum(end_point) ? nothing : DomainError("Vector domain error! Start point must smaller than end point")
    else
        ErrorException("Please input correct point you wan tto compute")
    end

    #The fractional integral of number is relating with the end_point.
    if isa(f, Number)
        f == 0 ? 0 : 2*f*sqrt(end_point/pi)
    end
end

function fracint(f::Union{Function, Number}, α, start_point, end_point::Real, h, ::RL_Direct)
    #checks(f, α, start_point, end_point)

    temp1 = f(start_point) .* (end_point-start_point) .^α
    #Use Complex differentiation to obtain the differentiation
    g(τ) = imag(f(τ .+ 1*im*h) ./ h) .* (end_point-τ) .^α
    temp2 = quadgk(g, start_point, end_point)
    result = (temp1 .+ temp2) ./gamma(α+1)
    return result
end

function fracint(f::Function, fd::Function, α, start_point, end_point, ::RL_Direct_First_Diff_Known)
    #checks(f, α, start_point, end_point)

    #The fractional integral of number is relating with the end_point.
    if typeof(f) <: Number
        if f == 0
            return 0
        else
            return 2*f*sqrt(end_point/pi)
        end
    end

    temp1 = f(start_point) .* (end_point-start_point) .^α
    g(τ) = fd(τ) .* (end_point-τ) .^α
    temp2 = quadgk(g, start_point, end_point)
    result = (temp1 .+ temp2) ./ gamma(α+1)
    return result
end

function fracint(f::Union{Function, Number}, α::Float64, start_point, end_point::AbstractArray, ::RL_Direct_First_Diff_Known)::Vector
    ResultArray = Float64[]

    for (_, value) in enumerate(end_point)
        append!(ResultArray, fracint(f, α, start_point, value, RL_Direct_First_Diff_Known())[1])
    end
    return ResultArray
end

function fracint(f::Union{Function, Number}, α::Float64, end_point, h, ::RL_Piecewise)::Float64
    #checks(f, α, 0, end_point)

    #Init
    n = Int64(floor(end_point/h))
    result = zero(Float64)

    for i ∈ 0:n
        result += W(i, n, α)*f(i*h)
    end

    result1 = result*h^α/gamma(α+2)
    return result1
end
function W(i, n, α)
    if i == 0
        return n^α*(α+1-n) + (n-1)^(α+1)
    elseif i == n
        return 1
    else
        return (n-i-1)^(α+1) + (n-i+1)^(α+1) - 2*(n-i)^(α+1)
    end
end

function fracint(f::Union{Number, Function}, α::Float64, end_point::AbstractArray, h, ::RL_Piecewise)::Vector
    result = map(x->fracint(f, α, x, h, RL_Piecewise()), end_point)
    return result
end

function fracint(f::Union{Function, Number}, α::Float64, end_point, h, ::RLInt_Approx)::Float64
    #checks(f, α, 0, end_point)

    α = -α
    n = Int64(floor(end_point/h))
    result = zero(Float64)

    for i ∈ 0:n-1
        result += (f(end_point - i*h) + f(end_point - (i+1)*h))/2*((i+1)^(-α) - i^(-α))
    end

    result1 = result*end_point^(-α)*n^α/gamma(1-α)
    return result1
end

function fracint(f::Union{Function, Number}, α::Float64, end_point::AbstractArray, h, ::RLInt_Approx)::Vector
    ResultArray = Float64[]

    for (_, value) in enumerate(end_point)
        append!(ResultArray, fracint(f, α, value, h, RLInt_Approx())[1])
    end
    return ResultArray
end

function fracint(f::Union{Number, Function}, α::Float64, end_point::AbstractArray, h, ::RLInt_Approx)::Vector
    result = map(x->fracint(f, α, x, h, RLInt_Approx()), end_point)
    return result
end

function fracint(f::Union{Function, Number}, α::Float64, end_point::Number, h, ::RL_LinearInterp)::Float64

    α = -α
    n = Int64(floor(end_point/h))
    result = zero(Float64)

    for i ∈ 0:n-1
        result += ((i+1)*f(end_point-i*h)-i*f(end_point-(i+1)*h))/(-α)*((i+1)^(-α) - i^(-α))+(f(end_point - (i+1)*h) - f(end_point-i*h))/(1-α)*((i+1)^(1-α) - i^(1-α))
    end

    result1 = result*end_point^(-α)*n^α/gamma(-α)
    return result1
end

function fracint(f::Union{Number, Function}, α::Float64, end_point::AbstractArray, h, ::RL_LinearInterp)::Vector
    result = map(x->fracint(f, α, x, h, RL_LinearInterp()), end_point)
    return result
end

function fracint(f, α::Number, end_point, h::Float64, ::RLInt_Matrix)
    N = Int64(floor(end_point/h+1))
    tspan = collect(0:h:end_point)
    return J(N, α, h)*f.(tspan)
end

function omega(n, p)
    omega = zeros(n+1)

    omega[1]=1
    @fastmath @inbounds @simd for i ∈ 1:n
        omega[i+1]=(1-(p+1)/i)*omega[i]
    end

    return omega

end
function J(N, p, h::Float64)
    result=zeros(N, N)
    temp=omega(N, -p)

    @fastmath @inbounds @simd for i ∈ 1:N
        result[i, 1:i]=reverse(temp[1:i])
    end

    return h^p*result
end

#=
RLInt_Simpson Algorithm
=#
function fracint(f, α, point, h, ::RLInt_Simpson)
    N = Int64(floor(point/h))
    temp1 = 0
    temp2 = 0

    @fastmath @inbounds @simd for i ∈ 0:N
        temp1 += cₖₙ(i, N, α)*f(i*h)
    end

    @fastmath @inbounds @simd for k ∈ 0:N-1
        temp2 += ĉₖₙ(k, N, α)*f((k+0.5)*h)
    end

    return h^α/gamma(α+3)*temp1 + 4*h^α/gamma(α+3)*temp2
end

function cₖₙ(k, n, α)
    if k == 0
        return 4*((n+1)^(2+α)-n^(2+α))-(α+2)*(3*(n+1)^(1+α)+n^(1+α))+(α+2)*(α+1)*(n+1)^α
    elseif 1 ≤ k ≤ n-1
        return -(α+2)*((n+1-k)^(1+α)+6*(n-k)^(1+α)+(n-k-1)^(1+α))+4*((n+1-k)^(2+α)-(n-1-k)^(2+α))
    elseif k == n
        return 2-α
    end
end
function ĉₖₙ(k, n, α)
    return ((α+2)*((n+1-k)^(1+α)+(n-k)^(1+α))-2*((n+1-k)^(2+α)-(n-k)^(2+α)))
end

function fracint(f::Union{Number, Function}, α::Float64, end_point::AbstractArray, h, ::RLInt_Simpson)::Vector
    result = map(x->fracint(f, α, x, h, RLInt_Simpson()), end_point)
    return result
end

#=
RLInt_Trapezoidal Algorithm
=#
function fracint(f, α, point, h, ::RLInt_Trapezoidal)
    N = Int64(floor(point/h))
    result = zero(Float64)

    @fastmath @inbounds @simd for i ∈ 0:N
        result += aₖₙ(i, N, α)*f(i*h)
    end

    return h^α/gamma(α+2)*result
end

function aₖₙ(k, n, α)
    if k == 0
        return (n-1)^(α+1) - (n-1-α)*n^α
    elseif 1 ≤ k ≤ n-1
        return (n-k+1)^(α+1)+(n-1-k)^(α+1)-2*(n-k)^(α+1)
    else
        return 1
    end
end

function fracint(f, α, point, h, ::RLInt_Rectangular)
    N=Int64(floor(point/h))
    result = zero(Float64)

    @fastmath @inbounds @simd for i ∈ 0:N-1
        result += bₖ(N-i-1, α)*f(i*h)
    end

    return h^α/gamma(α+1)*result
end

function bₖ(k, α)
    return (k+1)^α-k^α
end

function fracint(f::Union{Number, Function}, α::Float64, end_point::AbstractArray, h, ::RLInt_Rectangular)::Vector
    result = map(x->fracint(f, α, x, h, RLInt_Rectangular()), end_point)
    return result
end

#=
RLInt_Cubic_Spline_Interp Algorithm, when h is 0.01 behave best
=#
function fracint(f, α, point, h, ::RLInt_Cubic_Spline_Interp)
    N = Int64(floor(point/h))
    result = zero(Float64)

    @fastmath @inbounds @simd for j ∈ 0:N
        result += eⱼₙ(j, N, α)*f(j*h) + h*êⱼₙ(j, N, α)*first_order(f, j*h, h)
    end

    return h^α/gamma(α+4)*result
end

function eⱼₙ(j, n, α)
    if j == 0
        return -6*(n-1)^(2+α)*(1+2*n+α) + n^α*(12*n^3-6*(3+α)*n^2 + (1+α)*(2+α)*(3+α))
    elseif j == n
        return 6*(1+α)
    else
        return 6*(4*(n-j)^(3+α) + (n-j-1)^(2+α)*(2*j-2*n-1-α) + (1+n-j)^(2+α)*(2*j-2*n+1+α))
    end
end
function êⱼₙ(j, n, α)
    if j == 0
        return -2*(n-1)^(2+α)*(3*n+α) + n^(1+α)*(6*n^2-4*(3+α)*n + (2+α)*(3+α))
    elseif j == n
        return -2*α
    else
        return 2*(n-j-1)^(α+2)*(3*j-3*n-α)-2*(n-j+1)^(α+2)*(3*j-3*n+α) - (n-j)^(α+2)*(24+8α)
    end
end

function fracint(f::Union{Number, Function}, α::Float64, end_point::AbstractArray, h, ::RLInt_Cubic_Spline_Interp)::Vector
    result = map(x->fracint(f, α, x, h, RLInt_Cubic_Spline_Interp()), end_point)
    return result
end
# Deploy Complex Step Differentiation to compute the first order derivative.
function first_order(f, point, h)
    return imag(f(point + im*h))/h
end


module EqualityConstrainedMinimization
export Solve_ECM
using Calculus
using LinearAlgebra
include("algorithms.jl")

#######################################################################
#######################################################################
#=

This is a non-linear minimization solver that works with all convex problems
with linear equality constraints. It is intended to solve problems in the form
"minimize f(x) such that Ax = b" and "minimize f(x)" for the unconstrained case.
I am taking the algorithms used directly from "Convex Optimization" by Stephen Boyd
and Lieven Vandenberghe. This package is intended for learning purposes and while
it may not be the fastest solver, it is certainly one of the easiest to understand.

Algorithms used for unconstrained problem:
    -Gradient descent
    -Newton's method
    -Unless force_gradient = true, the solver will use gradient descent method until
     stopping criterion η is met. Then, it will switch to Newton's Method until ϵ is met
     Note that η > ϵ.

Algorithms used for equality constrained problem:
    -Feasible Start Newton Method
    -Infeasible Start Newton Method

# Arguments
    - 'f': The convex objective. Must be convex or the solution may converge to a local solution.
    - 'x': The starting point. May be feasible or infeasible though starting feasible may converge faster.
    - 'A': The coefficient matrix on the Ax = b constraint. This system needs to be underdetermined. Not including this will imply an unconstrained problem.
    - 'b': The constant vector on the Ax = b constraint. Not including this will imply feasible start.
    - 'α': A parameter used in the backtracking line search. Note that α ∈ (0, 0.5)
    - 'β': Another parameter used in the backtracking line search. β ∈ (0, 1)
    - 'η': A stopping criterion used to determine when to switch from gradient descent to Newton's method.
    - 'ϵ': Overall stopping criterion.
    - 'maxIt': Max iteration stopping criterion. Meant as a failsafe.
    - 'verbose': Setting this to true will show data from each iteration.
    - 'force_gradient': Setting this to true will force only the gradient descent method for unconstrained problems. Do this when ϵ is relatively large.

x^2 + y^2 -> x[1]^2 + x[2]^2

Main solver function:
Solve_ECM(
        f,
        x::Array{Float64,1};
        A::Array{Float64,2} = zeros(2, 2),
        b::Array{Float64,1} = zeros(2),
        α = 0.3,
        β = 0.8,
        η = 1e-4,
        ϵ = 1e-8,
        maxIt = 1e3,
        verbose = true,
        force_gradient = false
    )

See 'UsageExamples.jl' for some examples.

=#
#######################################################################
#######################################################################

#####################################
#####################################
#
# Main Solver
#
#####################################
#####################################
function Solve_ECM(
        f,
        x::Array{Float64,1};
        A::Array{Float64,2} = zeros(2, 2),
        b::Array{Float64,1} = zeros(2),
        α = 0.3,
        β = 0.8,
        η = 1e-4,
        ϵ = 1e-8,
        maxIt = 1e3,
        verbose = true,
        force_gradient = false
    )

    n = length(x)
    m = size(A,1)

    uncon = true
    feasible = true

    if η >= ϵ && n >= m

        if A != zeros(2, 2)
            if b != zeros(2)
                feasible = false
            end
            uncon = false
        end

        sol = Tuple{Array{Float64,1},Float64}

        #Determine which method to use
        if uncon

            print("\n\nUnconstrained objective detected")

            if force_gradient
                print("\nSolving using gradient descent\n")
                sol = Unconstrained_Grad_Descent(f, x, n, α, β, ϵ, maxIt, verbose)
            else
                print("\nSolving using gradient descent and Newton's Method\n")
                sol = Solve_UMP(f, x, n, α, β, η, ϵ, maxIt, verbose)
            end
        else
            if feasible
                print("\n\nVector b not given. Therefore, x is assumed feasible")
                print("\nSolving using Feasible Start Newton Method")
                sol = Newton_FS(f, x, A, n, m, α, β, ϵ, maxIt, verbose)
            else
                print("\n\nVector b given. Therefore, x is assumed infeasible")
                print("\nSolving using Infeasible Start Newton Method")
                sol = Newton_IFS(f, x, A, b, n, m, α, β, ϵ, maxIt, verbose)
            end
        end

        print("\n\n------------------------")
        print("\nSolution found at point:")
        print("\n\nx = ", sol[1],"\n")
        print("\nObjective Value = ", sol[2], "\n\n")
    else
        print("\nParameters not correct. Either η < ϵ or Ax = b is an overdetermined system")
    end

    return sol[2]
end

end


using Plots
using Polynomials
using DataFrames, DataFramesMeta
using Latexify
texpyplot()

using Random; Random.seed!(1234)
x = rand(-10:20,10)
y = 2 * x.^2 .+ rand(-80:80,length(x))
#x = [-3, 4, 5, -1, 2, -1, 0, -3, 3, -2]
#y = [-6, 6, 10, -1, 5, -2, -2, -7, 4, -2]

x[end], y[end]

df = @linq DataFrame(x = x[1:end-1], y = y[1:end-1]) |>
    by(:x, y = mean(:y))

g = polyfit(df.x, df.y)

xs = range(minimum(x), maximum(x), length = 200)

plot(xs, g.(xs), legend = false, linewidth = 2, grid = true)
plot!(x[1:end-1],y[1:end-1], seriestype = :scatter, markercolor = :lightgreen, markersize = 8)

savefig(joinpath(@__DIR__,"overfit.pdf"))

linreg(x, y) = hcat(fill!(similar(x), 1), x) \ y
a,b = linreg(x[1:end-1], y[1:end-1])

plot(xs, b*xs .+ a, legend = false, linewidth = 2, grid = true)
plot!(x[1:end-1],y[1:end-1], seriestype = :scatter, markercolor = :lightgreen, markersize = 8)
savefig(joinpath(@__DIR__,"underfit.pdf"))

plot(xs, 2*xs.^2, legend = false, linewidth = 2, grid = true)
plot!(x[1:end-1],y[1:end-1], seriestype = :scatter, markercolor = :lightgreen, markersize = 8)
savefig(joinpath(@__DIR__,"justright.pdf"))

clipboard(mdtable(hcat(x,y)))
clipboard(string(x))
clipboard(string(y))


using ModelingToolkit, StaticArrays, LinearAlgebra
using DiffEqBase, SparseArrays
using Test
using NonlinearSolve
using ModelingToolkit: value

canonequal(a, b) = isequal(simplify(a), simplify(b))

# Define some variables
@parameters t σ ρ β
@variables x y z

function test_nlsys_inference(name, sys, vs, ps)
    @testset "NonlinearSystem construction: $name" begin
        @test Set(states(sys))     == Set(value.(vs))
        @test Set(parameters(sys)) == Set(value.(ps))
    end
end

# Define a nonlinear system
eqs = [0 ~ σ*(y-x),
       0 ~ x*(ρ-z)-y,
       0 ~ x*y - β*z]
@named ns = NonlinearSystem(eqs, [x,y,z], [σ,ρ,β], defaults = Dict(x => 2))
@test eval(toexpr(ns)) == ns
test_nlsys_inference("standard", ns, (x, y, z), (σ, ρ, β))
@test begin
    f = eval(generate_function(ns, [x,y,z], [σ,ρ,β])[2])
    du = [0.0, 0.0, 0.0]
    f(du, [1,2,3], [1,2,3])
    du ≈ [1, -3, -7]
end

# Now nonlinear system with only variables
@variables x y z
@parameters σ ρ β

# Define a nonlinear system
eqs = [0 ~ σ*(y-x),
       y ~ x*(ρ-z),
       β*z ~ x*y]
@named ns = NonlinearSystem(eqs, [x,y,z], [σ,ρ,β])
jac = calculate_jacobian(ns)
@testset "nlsys jacobian" begin
    @test canonequal(jac[1,1], σ * -1)
    @test canonequal(jac[1,2], σ)
    @test canonequal(jac[1,3], 0)
    @test canonequal(jac[2,1], ρ - z)
    @test canonequal(jac[2,2], -1)
    @test canonequal(jac[2,3], x * -1)
    @test canonequal(jac[3,1], y)
    @test canonequal(jac[3,2], x)
    @test canonequal(jac[3,3], -1 * β)
end
nlsys_func = generate_function(ns, [x,y,z], [σ,ρ,β])
jac_func = generate_jacobian(ns)
f = @eval eval(nlsys_func)

# Intermediate calculations
a = y - x
# Define a nonlinear system
eqs = [0 ~ σ*a,
       0 ~ x*(ρ-z)-y,
       0 ~ x*y - β*z]
@named ns = NonlinearSystem(eqs, [x,y,z], [σ,ρ,β])
nlsys_func = generate_function(ns, [x,y,z], [σ,ρ,β])
nf = NonlinearFunction(ns)
jac = calculate_jacobian(ns)

@test ModelingToolkit.jacobian_sparsity(ns).colptr == sparse(jac).colptr
@test ModelingToolkit.jacobian_sparsity(ns).rowval == sparse(jac).rowval

jac = generate_jacobian(ns)

prob = NonlinearProblem(ns,ones(3),ones(3))
sol = solve(prob,NewtonRaphson())
@test sol.u[1] ≈ sol.u[2]

@test_throws ArgumentError NonlinearProblem(ns,ones(4),ones(3))

@variables u F s a
eqs1 = [
        0 ~ σ*(y-x) + F,
        0 ~ x*(ρ-z)-u,
        0 ~ x*y - β*z,
        0 ~ x + y - z - u,
       ]

lorenz = name -> NonlinearSystem(eqs1, [x,y,z,u,F], [σ,ρ,β], name=name)
lorenz1 = lorenz(:lorenz1)
@test_throws ArgumentError NonlinearProblem(lorenz1, zeros(5))
lorenz2 = lorenz(:lorenz2)
@named connected = NonlinearSystem([s ~ a + lorenz1.x
                             lorenz2.y ~ s
                             lorenz1.F ~ lorenz2.u
                             lorenz2.F ~ lorenz1.u], [s, a], [], systems=[lorenz1,lorenz2])
@test_nowarn alias_elimination(connected)

# system promotion
using OrdinaryDiffEq
@variables t
D = Differential(t)
@named subsys = convert_system(ODESystem, lorenz1, t)
@named sys = ODESystem([D(subsys.x) ~ subsys.x + subsys.x], t, systems=[subsys])
sys = structural_simplify(sys)
u0 = [subsys.x => 1, subsys.z => 2.0]
prob = ODEProblem(sys, u0, (0, 1.0), [subsys.σ=>1,subsys.ρ=>2,subsys.β=>3])
sol = solve(prob, Rodas5())
@test sol[subsys.x] + sol[subsys.y] - sol[subsys.z] ≈ sol[subsys.u]
@test_throws ArgumentError convert_system(ODESystem, sys, t)

@parameters t σ ρ β
@variables x y z

# Define a nonlinear system
eqs = [0 ~ σ*(y-x),
       0 ~ x*(ρ-z)-y,
       0 ~ x*y - β*z]
@named ns = NonlinearSystem(eqs, [x,y,z], [σ,ρ,β])
np = NonlinearProblem(ns, [0,0,0], [1,2,3], jac=true, sparse=true)
@test calculate_jacobian(ns, sparse=true) isa SparseMatrixCSC

# issue #819
@testset "Combined system name collisions" begin
       function makesys(name)
           @parameters a
           @variables x f

           NonlinearSystem([0 ~ -a * x + f], [x,f], [a]; name)
       end

       function issue819()
           sys1 = makesys(:sys1)
           sys2 = makesys(:sys1)
           @test_throws ArgumentError NonlinearSystem([sys2.f ~ sys1.x, sys1.f ~ 0], [], [], systems = [sys1, sys2], name=:foo)
       end
       issue819()
end

# issue #1115
@testset "Extending a NonlinearSystem with no iv" begin
    @parameters a b
    @variables x y
    eqs1 = [
        0 ~ a * x
    ]
    eqs2 = [
        0 ~ b * y
    ]

    @named sys1 = NonlinearSystem(eqs1, [x], [a])
    @named sys2 = NonlinearSystem(eqs2, [y], [b])
    @named sys3 = extend(sys1, sys2)

    @test isequal(union(Set(parameters(sys1)), Set(parameters(sys2))), Set(parameters(sys3)))
    @test isequal(union(Set(states(sys1)), Set(states(sys2))), Set(states(sys3)))
    @test isequal(union(Set(equations(sys1)), Set(equations(sys2))), Set(equations(sys3)))
end

# observed variable handling
@variables t x(t) RHS(t)
@parameters τ
@named fol = NonlinearSystem([0  ~ (1 - x)/τ], [x], [τ]; observed=[RHS ~ (1 - x)/τ])
@test isequal(RHS, @nonamespace fol.RHS)
RHS2 = RHS
@unpack RHS = fol
@test isequal(RHS, RHS2)

# issue #1358
@variables t
@variables v1(t) v2(t) i1(t) i2(t)
eq = [
    v1 ~ sin(2pi*t)
    v1 - v2 ~ i1
    v2 ~ i2
    i1 ~ i2
]
@named sys = ODESystem(eq)
@test length(equations(structural_simplify(sys))) == 0


# L0 pseudo-norm (times a constant)

"""
  NormL0(λ::Real=1.0)

Returns the function `g(x) = λ*countnz(x)`, for a nonnegative parameter `λ ⩾ 0`.
"""

immutable NormL0{R <: Real} <: ProximableFunction
  lambda::R
  function NormL0(lambda::R)
    if lambda < 0
      error("parameter λ must be nonnegative")
    else
      new(lambda)
    end
  end
end

NormL0{R <: Real}(lambda::R=1.0) = NormL0{R}(lambda)

function (f::NormL0){T <: RealOrComplex}(x::AbstractArray{T})
  return f.lambda*countnz(x)
end

function prox!{T <: RealOrComplex}(f::NormL0, x::AbstractArray{T}, y::AbstractArray{T}, gamma::Real=1.0)
  countnzy = 0;
  gl = gamma*f.lambda
  for i in eachindex(x)
    over = abs(x[i]) > sqrt(2*gl);
    y[i] = over*x[i];
    countnzy += over;
  end
  return f.lambda*countnzy
end

fun_name(f::NormL0) = "weighted L0 pseudo-norm"
fun_dom(f::NormL0) = "AbstractArray{Real}, AbstractArray{Complex}"
fun_expr(f::NormL0) = "x ↦ λ countnz(x)"
fun_params(f::NormL0) = "λ = $(f.lambda)"

function prox_naive{T <: RealOrComplex}(f::NormL0, x::AbstractArray{T}, gamma::Real=1.0)
  over = abs.(x) .> sqrt(2*gamma*f.lambda);
  y = x.*over;
  return y, f.lambda*countnz(y)
end


using Random, Statistics, LinearAlgebra, Dates #Shipped with Julia
using Distributions, StatsBase #Core statistics
using CSV, DataFrames #Basic Data
using Plots, StatsPlots, LaTeXStrings, Measures #Plotting and Output
using HypothesisTests, KernelDensity, GLM, Lasso, Clustering, MultivariateStats #Statistical/ML methods
using Flux, Metalhead #Deep learning
using Combinatorics, SpecialFunctions, Roots #Mathematical misc.
using RDatasets, MLDatasets #Example datasets
#uncomment if using R:  using RCall #Interface with R

# fix the random seed for reproducibility of random number sequences
fix_seed!() = Random.seed!(42)
fix_seed!()

readdir("./data")

# common operations
data = rand(Normal(), 5)  # 5 numbers selected from normal distributions
length(data)
size(data)
typeof(data)

n = length(data)
sum(data)/n

+(data...)/n  # + is just a function

# running mean formula
mn = 0  # initial value
for i ∈ 1:length(data)
    mn = (1/i)*data[i] + ((i-1)/i)*mn
end
mn

"""

My mean function. Works like this (inside)

"""
function my_mean(input_data)
    mn = 0  # initial value
    for (i,d) ∈ enumerate(input_data)
        mn = (1/i)*input_data[i] + ((i-1)/i)*mn
    end
    return mn
end

my_mean(data)

data = rand(Normal(), n) + im*rand(Normal(), n)
mean(data)
my_mean(data) # it just works!

methods(my_mean)
length(methods(mean)) # there are many different dispatches of mean

@which mean(data)


using Printf
using BenchmarkTools

println("-------------------------------")
println(@sprintf "Compute the first four Munchausen numbers")
println("-------------------------------")

"""
    raisedto(x)

Raise x to the power of itself.
"""
function raisedto(x)
    if x == 0
       return 0
    else
       return x^x
    end
end

function find_munchausen_numbers()

   power_of_digits = [raisedto(i) for i in 0:9]

   num = 0
   i = 0
   while num < 4
         thisnumber = i
         # find the sum of the digits raised to themselves
         sumofpowers = 0
         while thisnumber > 0
               digit = thisnumber % 10
               sumofpowers += power_of_digits[digit+1]
               thisnumber ÷= 10
         end
         if i == sumofpowers
            num += 1
            println("Munchausen number: ", num, " ", i)
         end
         i += 1
   end
end

@btime find_munchausen_numbers()

println(" ")


# TESTS - the classical test statistics for linear restrictions
#
# written by M. Creel, March 10, 1998.  michael.creel@uab.es
# converted to Octave 13 Jan. 2003
# converted to Julia 25 Aug. 2017
#
# purpose: calculates qF, Wald, Score and Likelihood Ratio tests for
# linear model						 y=XB+e
# 									 e~N(0,sig^2*I_n)
# subject to linear restrictions  	 RB=r
#
# format: [qF, W, LR, S] = TestStatistics(y,x,R,r)
#
# inputs:
# 		 y: nx1 dependent variable
# 		 x: nxk regressor matrix
# 		 R: R above, a qxk matrix
# 		 r: r above, a qx1 vector
#
# returns:
#        qF: the qF statistic
# 		 W: the Wald statistic
# 		 S: the score statistic
# 		 LR: the likelihood ratio statistic
using Distributions
function TestStatistics(y, x, R, r)

    n,k = size(x)
    q = size(R,1)
	b = x\y
	xx_inv = inv(x'*x)
	P_inv = inv(R*xx_inv*R')
	b_r = b - xx_inv*R'*P_inv*(R*b-r)
    e = y - x*b
    ess = (e'*e)[1,1]
    e_r = y - x*b_r
    ess_r = (e_r' * e_r)[1,1]
    sigsqhat_ols = ess/(n-k)
    sigsqhat_mle = ess/(n)
    sigsqhat_mle_r = ess_r/(n)
    # F-test
    F = (ess_r-ess)/(q*sigsqhat_ols)
    # Wald test (uses unrestricted model's est. of sig^2
    W = (R*b-r)'*P_inv*(R*b-r)/sigsqhat_mle
    # Score test (uses restricted model's est. of sig^2
    P_x = x * xx_inv * x'
    S = e_r' * P_x * e_r/(sigsqhat_mle_r)
    # LR test
    lnl = -n/2*log(2*pi) - n/2*log(sigsqhat_mle) - e' * e/(2*sigsqhat_mle)
    lnl_r = -n/2*log(2*pi) - n/2*log(sigsqhat_mle_r) - e_r' * e_r/(2*sigsqhat_mle_r)
    LR = 2.0*(lnl-lnl_r)
	tests = [q*F[1], W[1], LR[1], S[1]]
	pvalues = ccdf(Chisq(q), tests)
    tests = [tests pvalues]
	TESTS = ["qF","Wald","LR","Score"]
	labels = ["Value","p-value"]
    prettyprint(tests, labels, TESTS)
    return F, W, LR, S
end


# Newton 1d
ok, x, it, normf = findroot(x->2x, 1)
@test typeof(x) ==Float64
@test abs(x) < 1e-7
@test normf < 1e-13
@test ok
@test it < 4

ok, x, it, normf = findroot(x->x^2, 1)
@test abs(x) < 1e-5
@test normf < 1e-10
@test ok

ok, x, it, normf = findroot(x->(x^2-1)^2, 0.5)
@test abs(x-1) < 1e-5
@test normf < 1e-10
@test ok

ok, x, it, normf = findroot(x->(x^2-1)^2, 0.5, NewtonMethod(atol=1e-14))
@test abs(x-1) < 1e-7
@test normf < 1e-14
@test ok

# Interpolation
ok, x, it, normf = findroot(x->2x, 1, InterpolationMethod())
@test typeof(x) ==Float64
@test abs(x) < 1e-7
@test normf < 1e-7
@test ok
@test it < 4

ok, x, it, normf = findroot(x->exp(x)-1, 0.1, InterpolationMethod())
@test abs(x) < 1e-7
@test normf < 1e-7
@test ok

# broken on Travis, gnuplot error
# ok, x, it, normf = findroot(x->exp(x)-1, 0.1, InterpolationMethod(), atol=1e-13)
# @test abs(x) < 1e-10
# @test normf < 1e-13
# @test ok

# Newton 2d
ok, x, it, normf = findroot(x->[x[1]^2,x[2]^2], [1.,1.])
@test length(x) == 2
@test norm(x) < 1e-4
@test normf < 1e-10
@test ok


"""
    lis(arr::Array{Int}, ::Val{:dp})

# Arguments:
- `arr`: sequence of integers

# Examples/Tests
```julia
julia> lis([3, 10, 2, 1, 20], Val(:dp))
[3, 10, 20]

julia> lis([2, 2, 3], Val(:dp))
[2, 3]
```

# Brief:
Given a sequence of integers, the function finds the longest, strictly increasing, subsequence in that sequence.

# For more information:
https://cp-algorithms.com/sequences/longest_increasing_subsequence.html

# Contributed by:
- [Igor Malheiros](https://github.com/igormalheiros)
"""

function lis(arr::Array{Int}, ::Val{:dp})
    len = length(arr)
    memo = ones(Int, len)
    p = zeros(Int, len)

    lis_arr = Int[]

    len == 0 && return lis_arr # if arr is empty

    lis_value = 1
    lis_pos = 1

    for i in 2:len
        for j in 1:i
            if arr[j] < arr[i] && memo[i] < memo[j] + 1
                memo[i] = memo[j] + 1
                p[i] = j
            end
        end
        # Updates the longest increasing value and position
        if memo[i] > lis_value
            lis_value = memo[i]
            lis_pos = i
        end
    end

    # Restoring
    while lis_pos != 0
        push!(lis_arr, arr[lis_pos])
        lis_pos = p[lis_pos]
    end

    reverse!(lis_arr)

    return lis_arr
end


# 1D linear convection problem

# Packages and inclusions
using ModelingToolkit,DiffEqOperators,DiffEqBase,LinearAlgebra,Test

# Tests

@testset "Test 00: Dt(u(t,x)) ~ -Dx(u(t,x))" begin
    # Parameters, variables, and derivatives
    @parameters t x
    @variables u(..)
    Dt = Differential(t)
    Dx = Differential(x)

    # 1D PDE and boundary conditions
    eq  = Dt(u(t,x)) ~ -Dx(u(t,x))
    bcs = [u(0,x) ~ (0.5/(0.2*sqrt(2.0*3.1415)))*exp(-(x  -0.75)^2/(2.0*0.2^2)),
           u(t,0) ~ 0.0,
           u(t,2) ~ 0.0]

    # Space and time domains
    domains = [t ∈ IntervalDomain(0.0,0.6),
               x ∈ IntervalDomain(0.0,2.0)]

    # PDE system
    pdesys = PDESystem(eq,bcs,domains,[t,x],[u])

    # Method of lines discretization
    dx = 2/80
    order = 1
    discretization = MOLFiniteDifference(dx,order)

    # Convert the PDE problem into an ODE problem
    prob = discretize(pdesys,discretization)

    # Solve ODE problem
    using OrdinaryDiffEq
    sol = solve(prob,Euler(),dt=.025,saveat=0.1)

    # Plot and save results
    # using Plots
    # plot(prob.space[2],Array(prob.extrapolation[1]*sol[:,1,1]))
    # plot!(prob.space[2],Array(prob.extrapolation[1]*sol[:,1,2]))
    # plot!(prob.space[2],Array(prob.extrapolation[1]*sol[:,1,3]))
    # plot!(prob.space[2],Array(prob.extrapolation[1]*sol[:,1,4]))
    # plot!(prob.space[2],Array(prob.extrapolation[1]*sol[:,1,5]))
    # plot!(prob.space[2],Array(prob.extrapolation[1]*sol[:,1,6]))
    # savefig("MOL_1D_Linear_Convection_Test00.png")

    # Test
    x_interval = domains[2].domain.lower+dx:dx:domains[2].domain.upper-dx
    u = @. (0.5/(0.2*sqrt(2.0*3.1415)))*exp(-(x_interval-(0.75+0.6))^2/(2.0*0.2^2))
    t_f = size(sol,3)

    @test sol[:,1,t_f] ≈ u atol = 0.1;

end

@testset "Test 01: Dt(u(t,x)) ~ -Dx(u(t,x)) + 0.01" begin

    # Parameters, variables, and derivatives
    @parameters t x
    @variables u(..)
    Dt = Differential(t)
    Dx = Differential(x)

    # 1D PDE and boundary conditions
    eq  = Dt(u(t,x)) ~ -Dx(u(t,x)) + 0.01
    bcs = [u(0,x) ~ (0.5/(0.2*sqrt(2.0*3.1415)))*exp(-(x  -0.75)^2/(2.0*0.2^2)),
           u(t,0) ~ 0.0,
           u(t,2) ~ 0.0]

    # Space and time domains
    domains = [t ∈ IntervalDomain(0.0,0.6),
               x ∈ IntervalDomain(0.0,2.0)]

    # PDE system
    pdesys = PDESystem(eq,bcs,domains,[t,x],[u])

    # Method of lines discretization
    dx = 2/80
    order = 1
    discretization = MOLFiniteDifference(dx,order)

    # Convert the PDE problem into an ODE problem
    prob = discretize(pdesys,discretization)

    # Solve ODE problem
    using OrdinaryDiffEq
    sol = solve(prob,Euler(),dt=.025,saveat=0.1)

    # Plot and save results
    # using Plots
    # plot(prob.space[2],Array(prob.extrapolation[1]*sol[:,1,1]))
    # plot!(prob.space[2],Array(prob.extrapolation[1]*sol[:,1,2]))
    # plot!(prob.space[2],Array(prob.extrapolation[1]*sol[:,1,3]))
    # plot!(prob.space[2],Array(prob.extrapolation[1]*sol[:,1,4]))
    # plot!(prob.space[2],Array(prob.extrapolation[1]*sol[:,1,5]))
    # plot!(prob.space[2],Array(prob.extrapolation[1]*sol[:,1,6]))
    # savefig("MOL_1D_Linear_Convection_Test01.png")

    # Test
    x_interval = domains[2].domain.lower+dx:dx:domains[2].domain.upper-dx
    u = @. (0.5/(0.2*sqrt(2.0*3.1415)))*exp(-(x_interval-(0.75+0.6))^2/(2.0*0.2^2))
    t_f = size(sol,3)
    @test sol[:,1,t_f] ≈ u atol = 0.1;

end

@testset "Test 02: Dt(u(t,x)) ~ -v*Dx(u(t,x))" begin
    # Parameters, variables, and derivatives
    @parameters t x v
    @variables u(..)
    Dt = Differential(t)
    Dx = Differential(x)

    v = 1.0

    # 1D PDE and boundary conditions
    eq  = Dt(u(t,x)) ~ -v*Dx(u(t,x))
    bcs = [u(0,x) ~ (0.5/(0.2*sqrt(2.0*3.1415)))*exp(-(x-0.75)^2/(2.0*0.2^2)),
           u(t,0) ~ 0.0,
           u(t,2) ~ 0.0]

    # Space and time domains
    domains = [t ∈ IntervalDomain(0.0,0.6),
               x ∈ IntervalDomain(0.0,2.0)]

    # PDE system
    pdesys = PDESystem(eq,bcs,domains,[t,x],[u])

    # Method of lines discretization
    dx = 2/80
    order = 1
    discretization = MOLFiniteDifference(dx,order)

    # Convert the PDE problem into an ODE problem
    prob = discretize(pdesys,discretization)

    # Solve ODE problem
    using OrdinaryDiffEq
    sol = solve(prob,Euler(),dt=.025,saveat=0.1)

    # Plot and save results
    # using Plots
    # plot(prob.space[2],Array(prob.extrapolation[1]*sol[:,1,1]))
    # plot!(prob.space[2],Array(prob.extrapolation[1]*sol[:,1,2]))
    # plot!(prob.space[2],Array(prob.extrapolation[1]*sol[:,1,3]))
    # plot!(prob.space[2],Array(prob.extrapolation[1]*sol[:,1,4]))
    # plot!(prob.space[2],Array(prob.extrapolation[1]*sol[:,1,5]))
    # plot!(prob.space[2],Array(prob.extrapolation[1]*sol[:,1,6]))
    # savefig("MOL_1D_Linear_Convection_Test02.png")

    # Test
    x_interval = domains[2].domain.lower+dx:dx:domains[2].domain.upper-dx
    u = @. (0.5/(0.2*sqrt(2.0*3.1415)))*exp(-(x_interval-(0.75+v*0.6))^2/(2.0*0.2^2))
    t_f = size(sol,3)
    @test sol[:,1,t_f] ≈ u atol = 0.1;
end

@testset "Test 03: Dt(u(t,x)) ~ -Dx(v(t,x))*u(t,x)-v(t,x)*Dx(u(t,x)) with v(t,x)=1" begin
    # Parameters, variables, and derivatives
    @parameters t x
    @variables v(..) u(..)
    Dt = Differential(t)
    Dx = Differential(x)

    # 1D PDE and boundary conditions
    eq  = [ Dt(u(t,x)) ~ -(Dx(v(t,x))*u(t,x)+v(t,x)*Dx(u(t,x))),
            v(t,x) ~ 1.0 ]
    bcs = [u(0,x) ~ (0.5/(0.2*sqrt(2.0*3.1415)))*exp(-(x-0.75)^2/(2.0*0.2^2)),
           u(t,0) ~ 0.0,
           u(t,2) ~ 0.0,
           v(0,x) ~ 1.0,
           v(t,0) ~ 1.0,
           v(t,2) ~ 1.0
           ]

    # Space and time domains
    domains = [t ∈ IntervalDomain(0.0,0.6),
               x ∈ IntervalDomain(0.0,2.0)]

    # PDE system
    pdesys = PDESystem(eq,bcs,domains,[t,x],[u,v])

    # Method of lines discretization
    dx = 2/80
    order = 1
    discretization = MOLFiniteDifference(dx,order)

    # Convert the PDE problem into an ODE problem
    prob = discretize(pdesys,discretization)

    # Solve ODE problem
    using OrdinaryDiffEq
    sol = solve(prob,Euler(),dt=.025,saveat=0.1)

    #Plot and save results
    # using Plots
    # plot(prob.space[2],Array(prob.extrapolation[1]*sol[:,1,1]))
    # plot!(prob.space[2],Array(prob.extrapolation[1]*sol[:,1,2]))
    # plot!(prob.space[2],Array(prob.extrapolation[1]*sol[:,1,3]))
    # plot!(prob.space[2],Array(prob.extrapolation[1]*sol[:,1,4]))
    # plot!(prob.space[2],Array(prob.extrapolation[1]*sol[:,1,5]))
    # plot!(prob.space[2],Array(prob.extrapolation[1]*sol[:,1,6]))
    # savefig("MOL_1D_Linear_Convection_Test03.png")

    # Test
    x_interval = domains[2].domain.lower+dx:dx:domains[2].domain.upper-dx
    u = @. (0.5/(0.2*sqrt(2.0*3.1415)))*exp(-(x_interval-(0.75+1.0*0.6))^2/(2.0*0.2^2))
    t_f = size(sol,3)

    @test sol[:,1,t_f] ≈ u atol = 0.1;
end

@testset "Test 04: Dt(u(t,x)) ~ -Dx(v(t,x))*u(t,x)-v(t,x)*Dx(u(t,x)) with v(t,x)=0.999 + 0.001 * t * x " begin
    # Parameters, variables, and derivatives
    @parameters t x
    @variables v(..) u(..)
    Dt = Differential(t)
    Dx = Differential(x)

    # 1D PDE and boundary conditions
    eq  = [ Dt(u(t,x)) ~ -Dx(v(t,x))*u(t,x)-v(t,x)*Dx(u(t,x)),
            v(t,x) ~ 0.999 + 0.001 * t * x ]
    bcs = [u(0,x) ~ (0.5/(0.2*sqrt(2.0*3.1415)))*exp(-(x-0.75)^2/(2.0*0.2^2)),
           u(t,0) ~ 0.0,
           u(t,2) ~ 0.0,
           v(0,x) ~ 0.999,
           v(t,0) ~ 0.999,
           v(t,2) ~ 0.999 + 0.001 * t * 2.0]

    # Space and time domains
    domains = [t ∈ IntervalDomain(0.0,0.6),
               x ∈ IntervalDomain(0.0,2.0)]

    # PDE system
    pdesys = PDESystem(eq,bcs,domains,[t,x],[u,v])

    # Method of lines discretization
    dx = 2/80
    order = 1
    discretization = MOLFiniteDifference(dx,order)

    # Convert the PDE problem into an ODE problem
    prob = discretize(pdesys,discretization)

    # Solve ODE problem
    using OrdinaryDiffEq
    sol = solve(prob,Euler(),dt=.025,saveat=0.1)

    # Plot and save results
    # using Plots
    # plot(prob.space[2],Array(prob.extrapolation[1]*sol[:,1,1]))
    # plot!(prob.space[2],Array(prob.extrapolation[1]*sol[:,1,2]))
    # plot!(prob.space[2],Array(prob.extrapolation[1]*sol[:,1,3]))
    # plot!(prob.space[2],Array(prob.extrapolation[1]*sol[:,1,4]))
    # plot!(prob.space[2],Array(prob.extrapolation[1]*sol[:,1,5]))
    # plot!(prob.space[2],Array(prob.extrapolation[1]*sol[:,1,6]))
    # savefig("MOL_1D_Linear_Convection_Test04.png")

    # Test
    x_interval = domains[2].domain.lower+dx:dx:domains[2].domain.upper-dx
    u = @. (0.5/(0.2*sqrt(2.0*3.1415)))*exp(-(x_interval-(0.75+1.0*0.6))^2/(2.0*0.2^2))
    t_f = size(sol,3)

    @test sol[:,1,t_f] ≈ u atol = 0.1;
end


module Poisson1DTests

using Gridap
using Test

u(x) = x[1]^2 + 1
∇u(x) = ∇(u)(x)
f(x) = -Δ(u)(x)

domain = (0,3)
cells = (10,)
model = CartesianDiscreteModel(domain,cells)

order = 2
V = FESpace(
  model=model,
  reffe=:Lagrangian,
  order=order,
  valuetype=Float64,
  conformity=:H1,
  dirichlet_tags=2)

U = TrialFESpace(V,u)

degree = 2*order
trian = Triangulation(model)
quad = CellQuadrature(trian,degree)

btrian = BoundaryTriangulation(model,1)
bquad = CellQuadrature(btrian,degree)
nb = get_normal_vector(btrian)

a(u,v) = ∇(v)⋅∇(u)
l(v) = v*f
t_Ω = AffineFETerm(a,l,trian,quad)

l_b(v) = v*(nb⋅∇u)
t_b = FESource(l_b,btrian,bquad)

op = AffineFEOperator(U,V,t_Ω,t_b)
uh = solve(op)

e = u - uh

l2(u) = inner(u,u)
sh1(u) = a(u,u)
h1(u) = sh1(u) + l2(u)

el2 = sqrt(sum( integrate(l2(e),trian,quad) ))
eh1 = sqrt(sum( integrate(h1(e),trian,quad) ))
ul2 = sqrt(sum( integrate(l2(uh),trian,quad) ))
uh1 = sqrt(sum( integrate(h1(uh),trian,quad) ))

@test el2/ul2 < 1.e-8
@test eh1/uh1 < 1.e-7

end # module


function normpdf(x::Float64, mu::Float64, sigma::Float64)
	return 1.0 / ( sqrt(2.0 * pi) * sigma ) * exp( - 0.5 * (x - mu)^2 / sigma^2 )
end


using Distributions
using LaTeXStrings

# Create plot of Beta(α,β) density
α = 10
β = 5

x_values = 0:0.01:1
y_values = [pdf(Beta(α,β),x) for x in x_values]

plot(x_values, y_values, seriestype = :line, legend = false,
 yaxis = L"f(\theta)", xaxis = L"\theta", grid = false)

savefig("beta_prior_coin_ex.svg")


# A Pythagorean triplet is a set of three natural numbers, a < b < c, for which,
#
# a^2 + b^2 = c^2
# For example, 3^2 + 4^2 = 9 + 16 = 25 = 5^2.
#
# There exists exactly one Pythagorean triplet for which a + b + c = 1000.
# Find the product abc.

using ProjectEulerSolutions

# Cycle through (from n down to n/2) to find sets of a and b that satisfy
# a + b + c = n.
function p009solution(n::Integer=4)::Integer

    maxval = fld(n, 2)
    for a = maxval:-1:2
        for b = maxval:-1:2
            c = sqrt(a*a + b*b)
            if c == floor(c) && a + b + c == n
                return a * b * c
            end
        end
    end

    # Should never happen
    return -1
end

# Note, there are probably faster options which use triplet generators.

p009 = Problems.Problem(p009solution)

Problems.benchmark(p009, 1000)


# Polynomial mean function

@doc """
# Description
Constructor for the Polynomial mean function

m(x) = Σ_d xᵈβᵀ_d
# Arguments:
* `β::Matrix{Float64}`: A Dxd matrix of coefficients where D is the dimension of xᵈ and d is the degree of the polynomial
""" ->
type MeanPoly <: Mean
    β::Matrix{Float64}    # Polynomial coefficients
    dim::Int              # Dimension
    deg::Int              # Polynomial degree
    MeanPoly(β::Matrix{Float64}) = new(β,size(β, 1), size(β, 2))
end

function mean(mPoly::MeanPoly,x::Matrix{Float64})
    dim, nobsv = size(x)
    dim == mPoly.dim || throw(ArgumentError("Observations and mean function have inconsistent dimensions"))
    z = zeros(nobsv)
    for i in 1:nobsv
        for j in 1:mPoly.deg
            z[i] += dot(x[:, i].^j, mPoly.β[:,j])
        end
    end
    return z
end

get_params(mPoly::MeanPoly) = vec(mPoly.β)
get_param_names(mPoly::MeanPoly) = get_param_names(mPoly.β, :β)
num_params(mPoly::MeanPoly) = mPoly.dim * mPoly.deg
function set_params!(mPoly::MeanPoly, hyp::Vector{Float64})
    num_param = mPoly.dim * mPoly.deg
    length(hyp) == num_param || throw(ArgumentError("Polynomial mean function has $(num_param) parameters"))
    mPoly.β = reshape(hyp,mPoly.dim,mPoly.deg)
end

function grad_mean(mPoly::MeanPoly, x::Vector{Float64})
    dM_theta = Array(Float64,mPoly.dim,mPoly.deg)

    for i in 1:mPoly.dim
        for j in 1:mPoly.deg
            dM_theta[i,j] = x[i].^j
        end
    end
    return vec(dM_theta)
end


using Printf
using Turing
using CairoMakie
using ElectronDisplay

# Data generation

N = 100
x = range(0, 1, length=N)
y = @. 0.16x^2 + 4.5x + π
y = @. y * (1 + 0.1 * randn())

scatter(x, y)

# Linear model (y = mx + c)

@model function linear_model(x, y)
    m ~ Normal(0, 1)
    c ~ Normal(0, 1)
    σ ~ Exponential(1)

    N = length(x)
    for n in 1:N
        y[n] ~ Normal(m*x[n] + c, σ)
    end
end

chain_linear = sample(linear_model(x, y), NUTS(0.65), 10000, progress=true)

function y_linear(chain, n, x)
    m = chain[:m][n]
    c = chain[:c][n]
    return @. m * x + c
end

function animate_chain(chain, y_model, x, y; filepath, N_samples=1000, N_history=50, max_alpha=0.5)
    n = Observable(1)

    # Compute all predictions to plot.
    yₚ = []
    for i in 1:N_samples
        push!(yₚ, y_model(chain, i, x))
    end

    fig = Figure()

    title = @lift "Iteration $($n)"
    ax = Axis(fig[1, 1], xlabel="x", ylabel="y"; title)

    # Create N_history lines with different alpha that we can change the data on.
    line_plots = []
    for i in 1:N_history
        line_plot = lines!(x, yₚ[1], color=(:gray, max_alpha * i/N_history))
        push!(line_plots, line_plot)
    end

    # Plot the actual data points.
    scatter!(x, y)

    record(fig, filepath, 1:N_samples, framerate=30) do chain_iter
        @info "Animating $filepath: frame $chain_iter/$N_samples"
        n[] = chain_iter

        # Change the data of the line plots to show the last N_history predictions.
        for (i, j) in enumerate(chain_iter:-1:max(1, chain_iter-N_history+1))
            line_plots[i][2] = yₚ[j]
        end
    end

    return
end

animate_chain(chain_linear, y_linear, x, y, filepath="linear_model.mp4")

function plot_chains_and_densities(chain, vars; filepath)
    fig = Figure(resolution=(800, 200*length(vars)))

    for (n, var) in enumerate(vars)
        ax1 = Axis(fig[n, 1], ylabel="$var", xlabel="iteration")
        lines!(ax1, chain[var][:] |> skipmissing |> collect)

        ax2 = Axis(fig[n, 2], ylabel="P($var)", xlabel="$var", yticklabelsvisible=false)
        density!(ax2, chain[var][:] |> skipmissing |> collect)
    end

    save(filepath, fig, px_per_unit=2)

    return fig
end

plot_chains_and_densities(chain_linear, [:m, :c, :σ], filepath="linear_model_posteriors.png")

# Quadratic model (y = ax² + bx + c)

@model function quadratic_model(x, y)
    a ~ Normal(0, 1)
    b ~ Normal(0, 1)
    c ~ Normal(0, 1)
    σ ~ Exponential(1)

    N = length(x)
    for n in 1:N
        y[n] ~ Normal(a*x[n]^2 + b*x[n] + c, σ)
    end
end

chain_quadratic = sample(quadratic_model(x, y), NUTS(0.65), 10000, progress=true)

function y_quadratic(chain, n, x)
    a = chain[:a][n]
    b = chain[:b][n]
    c = chain[:c][n]
    return @. a*x^2 + b*x + c
end

animate_chain(chain_quadratic, y_quadratic, x, y, filepath="quadratic_model.mp4")

plot_chains_and_densities(chain_quadratic, [:a, :b, :c, :σ], filepath="quadratic_model_posteriors.png")

# Exponential model [y = α + β * exp(γ * x)]

@model function exponential_model(x, y)
    α ~ Normal(0, 1)
    β ~ Normal(0, 1)
    γ ~ Normal(0, 1)
    σ ~ Exponential(1)

    N = length(x)
    for n in 1:N
        y[n] ~ Normal(α + β * exp(γ*x[n]), σ)
    end
end

chain_exponential = sample(exponential_model(x, y), NUTS(0.65), 10000, progress=true)

function y_exponential(chain, n, x)
    α = chain[:α][n]
    β = chain[:β][n]
    γ = chain[:γ][n]
    return @. α + β * exp(γ * x)
end

animate_chain(chain_exponential, y_exponential, x, y, filepath="exponential_model.mp4")

plot_chains_and_densities(chain_exponential, [:α, :β, :γ, :σ], filepath="exponential_model_posteriors.png")

# Categorical model (picks between linear, quadratic, and exponential)

@model function categorical_model1(x, y)
    N = length(x)

    pl ~ Beta(1, 1)
    M ~ Categorical([pl, 1-pl])

    if M == 1
        m ~ Normal(0, 1)
        k ~ Normal(0, 1)
        σl ~ Exponential(1)

        for n in 1:N
            y[n] ~ Normal(m*x[n] + k, σl)
        end
    else
        a ~ Normal(0, 1)
        b ~ Normal(0, 1)
        c ~ Normal(0, 1)
        σq ~ Exponential(1)

        for n in 1:N
            y[n] ~ Normal(a*x[n]^2 + b*x[n] + c, σq)
        end
    end
end

prior_chain_categorical1 = sample(categorical_model1(x, y), Prior(), 10000)
plot_chains_and_densities(prior_chain_categorical1, [:pl, :M, :a, :b, :c, :m, :k], filepath="categorical_model1_priors.png")

# Errors since variables defined in if statements are missing during inference?
sampler = Gibbs(PG(100, :M), NUTS(100, 0.65, :pl, :m, :k, :σl, :a, :b, :c, :σq))
chain_categorical1 = sample(categorical_model1(x, y), sampler, 1000, progress=true)

@model function categorical_model2(x, y)
    N = length(x)

    pl ~ Beta(1, 1)
    M ~ Categorical([pl, 1-pl])

    m ~ Normal(0, 1)
    k ~ Normal(0, 1)
    σl ~ Exponential(1)

    a ~ Normal(0, 1)
    b ~ Normal(0, 1)
    c ~ Normal(0, 1)
    σq ~ Exponential(1)

    if M == 1
        for n in 1:N
            y[n] ~ Normal(m*x[n] + k, σl)
        end
    else
        for n in 1:N
            y[n] ~ Normal(a*x[n]^2 + b*x[n] + c, σq)
        end
    end
end

prior_chain_categorical2 = sample(categorical_model2(x, y), Prior(), 10000)
plot_chains_and_densities(prior_chain_categorical2, [:pl, :M, :a, :b, :c, :m, :k], filepath="categorical_model2_priors.png")

# Inference fails? since it always picks M=1 or M=2 and doesn't sample from both models.
sampler = Gibbs(PG(10, :M), NUTS(100, 0.65, :pl, :m, :k, :σl, :a, :b, :c, :σq))
chain_categorical2 = sample(categorical_model2(x, y), sampler, 1000, progress=true)
plot_chains_and_densities(chain_categorical2, [:pl, :M, :m, :k, :σl, :a, :b, :c, :σq], filepath="categorical_model2_posteriors.png")

@model function categorical_model3(x, y)
    N = length(x)

    m ~ Normal(0, 1)
    k ~ Normal(0, 1)
    σl ~ Exponential(1)

    a ~ Normal(0, 1)
    b ~ Normal(0, 1)
    c ~ Normal(0, 1)
    σq ~ Exponential(1)

    pl ~ Beta(1, 1)

    if rand() < pl
        for n in 1:N
            y[n] ~ Normal(m*x[n] + k, σl)
        end
    else
        for n in 1:N
            y[n] ~ Normal(a*x[n]^2 + b*x[n] + c, σq)
        end
    end
end

# Inference is bad and chain is very correlated.
chain_categorical3 = sample(categorical_model3(x, y), NUTS(), 1000, progress=true)
plot_chains_and_densities(chain_categorical3, [:pl, :m, :k, :σl, :a, :b, :c, :σq], filepath="categorical_model3_posteriors.png")

function plot_model_comparison(chain, x, y)
    fig = Figure()
    ax = Axis(fig[1, 1])

    pl = mode(chain[:pl])

    m = mode(chain[:m])
    k = mode(chain[:k])

    a = mode(chain[:a])
    b = mode(chain[:b])
    c = mode(chain[:c])

    y_linear = @. m*x + k
    y_quadratic = @. a*x^2 + b*x + c

    lines!(ax, x, y_linear, label=@sprintf("linear (%d%%)", 100pl))
    lines!(ax, x, y_quadratic, label=@sprintf("quadratic (%d%%)", 100*(1-pl)))

    scatter!(x, y)
    axislegend(ax)

    return fig
end

plot_model_comparison(chain_categorical3, x, y)

chain_time(chain) = chain.info.stop_time - chain.info.start_time

chains = (chain_linear, chain_quadratic, chain_exponential, chain_categorical2, chain_categorical3)
chain_names = ("linear", "quadratic", "exponential", "categorical1", "categorical2")

open("inference_wall_clock_times.txt","w") do io
    for (chain, name) in zip(chains, chain_names)
        line = @sprintf("%s: %.1f seconds for %d samples", name, chain_time(chain), size(chain, 1))
        @info line
        println(io, line)
    end
end


using LinearAlgebra

function cheb(N)
    if N==0
        D=0; x=1
        return D,x
    else
        x= cos.(pi * collect(0:N) ./ N)
        c=[2; ones(N-1, 1); 2] .* (-1).^collect(0:N);
        X = repeat(x, 1, N+1);
        dX=X .- X';
        D = (c .* (1 ./ c)')  ./ (dX .+ Diagonal( ones(N+1) ) );
        D = D .- Diagonal( vec( sum(D', dims=1) ) );
        return D,x
    end
end

function clencurt(N)

     θ = π * collect(0:N) ./N
     x = cos.(θ)
     w = zeros(1, N+1)
    ii = collect(2:N)
     v = ones(N-1,1)

    if mod(N,2) == 0
          w[1] = 1/(N^2 - 1)
        w[N+1] = w[1]
        for k=1:N/2-1
             v = v .- 2 * cos.(2 * k * θ[ii])/(4 * k^2 - 1)
        end
        v = v - cos.(N*θ[ii]) / (N^2 - 1)
    else
          w[1] = 1/N^2
        w[N+1] = w[1]
        for k=1:(N-1)/2
            v = v .- 2 * cos.(2 * k * θ[ii])/(4 * k^2 - 1)
        end
    end

    w[ii] = 2*v/N

    return x, w
end


export cg1

"""
`cg1(j1, m1, j2, m2, j3, m3, T=Float64)` : A reference implementation of
Clebsch-Gordon coefficients based on

https://hal.inria.fr/hal-01851097/document
Equation (4-6)

This heavily uses BigInt and BigFloat and should therefore not be employed
for performance critical tasks.
"""
function cg1(j1, m1, j2, m2, j3, m3, T=Float64)
   if (m3 != m1 + m2) || !(abs(j1-j2) <= j3 <= j1 + j2)
      return zero(T)
   end

   N = (2*j3+1) *
       factorial(big(j1+m1)) * factorial(big(j1-m1)) *
       factorial(big(j2+m2)) * factorial(big(j2-m2)) *
       factorial(big(j3+m3)) * factorial(big(j3-m3)) /
       factorial(big( j1+j2-j3)) /
       factorial(big( j1-j2+j3)) /
       factorial(big(-j1+j2+j3)) /
       factorial(big(j1+j2+j3+1))

   G = big(0)
   # 0 ≦ k ≦ j1+j2-j3
   # 0 ≤ j1-m1-k ≤ j1-j2+j3   <=>   j2-j3-m1 ≤ k ≤ j1-m1
   # 0 ≤ j2+m2-k ≤ -j1+j2+j3  <=>   j1-j3+m2 ≤ k ≤ j2+m2
   lb = (0, j2-j3-m1, j1-j3+m2)
   ub = (j1+j2-j3, j1-m1, j2+m2)
   for k in maximum(lb):minimum(ub)
      bk = big(k)
      G += (-1)^k *
           binomial(big( j1+j2-j3), big(k)) *
           binomial(big( j1-j2+j3), big(j1-m1-k)) *
           binomial(big(-j1+j2+j3), big(j2+m2-k))
   end

   return T(sqrt(N) * G)
end


# v0.6

# Task 1
function hofstadterconway(n::Integer)::Vector{Int}
    rst = fill(1, n)
    for i in 3:n
        rst[i] = rst[rst[i - 1]] + rst[i - rst[i - 1]]
    end
    return rst
end

function hcfraction(n::Integer)::Vector{Float64}
    rst = Array{Float64}(n)
    for (i, a) in enumerate(hofstadterconway(n))
        rst[i] = abs(a) / i
    end
    return rst
end

# Task 2
seq = hcfraction(2 ^ 20)
for i in 1:20
    a, b = 2 ^ (i - 1), 2 ^ i
    @printf("max value of a(n)/n in %6i < n < %7i = %5.3f\n", a, b, maximum(seq[a:b]))
end

# Task 3
function lastindex(val::Float64)
    r, p = 1, 0
    # Find the range of 2 power in which the maximum is < val
    seq = hcfraction(2 ^ 15)
    while maximum(seq[2^p:2^(p+1)]) > val; p += 1 end
    r = 2 ^ (p + 1)
    while seq[r] < val; r -= 1 end
    return r
end

println("You too might have won \$1000 with the mallows number of ", lastindex(0.55))


a=rand(Int,3).%9;b=rand(Int,3).%9;c=rand(Int,3).%9
abc=[Equation(:a,a),Equation(:b,b),Equation(:c,c)]

#Dot product
ex1=Ten(:a,:i)*Ten(:b,:i)
r1=ex1&abc
@assert r1==dot(a,b)

#Cross product
ex2=Alt([:i,:j,:k])*Ten(:a,:j)*Ten(:b,:k)
r2=ex2&abc
@assert r2.x==cross(a,b)

ex3=Alt([:i,:j,:k])*Ten(:a,:j)*Ten(:b,:k)*Ten(:b,:m)*Ten(:a,:m)+Ten(:c,:i)
r3=ex3&abc
@assert r3.x==cross(a,b)*dot(b,a)+c

#Both
ex4=Alt([:i,:j,:k])*Ten(:a,:j)*Ten(:b,:k)*Ten(:a,:m)*Ten(:c,:m)
r4=ex4&abc
@assert r4.x==cross(a,b)*dot(a,c)

ex5=Alt([:i,:j,:k])*Ten(:a,:j)*Ten(:b,:k)*Alt([:m,:n,:o])*Ten(:c,:n)*Ten(:b,:o)
r5=ex5&abc&Equation(:m,:i)
@assert r5==dot(cross(a,b),cross(c,b))

#Triangle area
ex6=Sqrt(Alt([:i,:j,:k])*Ten(:a,:j)*Ten(:b,:k)*Alt([:i,:m,:n])*Ten(:a,:m)*Ten(:b,:n))/2
r6=ex6&abc
@assert round(r6)==round(norm(cross(a,b))/2)

#Diadic products
ex7=Ten(:a,:i)*Ten(:b,:j)
r7=ex7&abc&@equs(i=2, j=3)
@assert r7==a[2]*b[3]

#Transpose
ex8=Equations.Transpose(Ten(a*b',[:i,:j]))
r8=simplify(ex8)
@assert r8.x==(a*b')'

#Determinant
A=rand(Int,3,3).%9
ex9a=Alt([:i,:j,:k])*Ten(A,[:i,1])*Ten(A,[:j,2])*Ten(A,[:k,3])
r9a=simplify(ex9a)
ex9b=Alt([:i,:j,:k])*Alt([:r,:s,:t])*Ten(A,[:i,:r])*Ten(A,[:j,:s])*Ten(A,[:k,:t])/6
r9b=simplify(ex9b)
@assert round(det(A))==r9a==round(r9b)

#Inverse
eq1=@equ detA=Alt([x,y,z])*Ten(A,[x,1])*Ten(A,[y,2])*Ten(A,[z,3])
eq2=@equ invA=Alt([j,m,n])*Alt([i,p,q])*Ten(A,[m,p])*Ten(A,[n,q])/(2detA)
eq3=eq2&eq1&Equation(:A,A)
@assert abs((eq3&@equs(i=1,j=3)).rhs-inv(A)[1,3])<0.001

#Matrix multiplication
B=rand(Int,3,3).%9
AB=[Equation(:A,A),Equation(:B,B)]
C=Ten(:A,[:i,:k])*Ten(:B,[:k,:j])
@assert C&AB&@equs(i=3,j=2) == (A*B)[3,2]

D=Ten(:A,[:i,:j])*Ten(:B,[:i,:j])
@assert D&AB==sum(A.*B)


# Centroid
# ---------------------------------------------------------------------------------------------
# (https://en.wikipedia.org/wiki/Centroid#Of_a_polygon)
function centroid(poly::Polygon{N, 2, T}) where {N, T}
    a = zero(T) # Scalar
    c = SVector{2,T}(0,0)
    for i ∈ 1:N-1
        subarea = poly[i] × poly[i+1]
        c += subarea*(poly[i] + poly[i+1])
        a += subarea
    end
    subarea = poly[N] × poly[1]
    c += subarea*(poly[N] + poly[1])
    a += subarea
    return Point(c/(3a))
end

centroid(tri::Triangle2D) = Point((tri[1] + tri[2] + tri[3])/3)
centroid(tri::Triangle3D) = Point((tri[1] + tri[2] + tri[3])/3)

centroid(quad8::QuadraticQuadrilateral2D) = centroid(quad8, Val(3))
function centroid(quad8::QuadraticQuadrilateral{Dim, T}, ::Val{N}) where {Dim, T, N}
    # Gauss-Legendre quadrature over a quadrilateral is used.
    # Let Q(r,s) be the interpolation function for quad8,
    #            1  1                        N   N
    # A = ∬ dA = ∫  ∫ ‖∂Q/∂r × ∂Q/∂s‖ds dr = ∑   ∑ wᵢwⱼ‖∂Q/∂r(rᵢ,sⱼ) × ∂Q/∂s(rᵢ,sⱼ)‖
    #     S      0  0                       j=1 i=1
    #                  1  N   N
    # 𝗖 = (∬ 𝘅 dA)/A = -  ∑   ∑ 𝘅ᵢⱼwᵢwⱼ‖∂Q/∂r(rᵢ,sⱼ) × ∂Q/∂s(rᵢ,sⱼ)‖
    #      S           A j=1 i=1
    w, r = gauss_legendre_quadrature(T, Val(N))
    A = zero(T)
    𝗖 = @SVector zeros(T, Dim)
    for j ∈ 1:N, i ∈ 1:N
        J = 𝗝(quad8, r[i], r[j])
        weighted_val = w[i]*w[j]*norm(view(J, :, 1) × view(J, :, 2))
        𝗖 += weighted_val * quad8(r[i], r[j]).coord
        A += weighted_val
    end
    return Point(𝗖)/A
end

centroid(tri6::QuadraticTriangle2D) = centroid(tri6, Val(6))
function centroid(tri6::QuadraticTriangle{Dim, T}, ::Val{N}) where {Dim, T, N}
    # Gauss-Legendre quadrature over a triangle is used.
    # Let F(r,s) be the interpolation function for tri6,
    #            1 1-r                       N
    # A = ∬ dA = ∫  ∫ ‖∂F/∂r × ∂F/∂s‖ds dr = ∑ wᵢ‖∂F/∂r(rᵢ,sᵢ) × ∂F/∂s(rᵢ,sᵢ)‖
    #     S      0  0                       i=1
    #                  1  N
    # 𝗖 = (∬ 𝘅 dA)/A = -  ∑ 𝘅 wᵢ‖∂F/∂r(rᵢ,sᵢ) × ∂F/∂s(rᵢ,sᵢ)‖
    #      S           A i=1
    w, r, s = gauss_legendre_quadrature(tri6, Val(N))
    A = zero(T)
    𝗖 = @SVector zeros(T, Dim)
    for i ∈ 1:N
        J = 𝗝(tri6, r[i], s[i])
        weighted_val = w[i] * norm(view(J, :, 1) × view(J, :, 2))
        𝗖 += weighted_val * tri6(r[i], s[i])
        A += weighted_val
    end
    return Point(𝗖)/A
end


"""
    BinLik <: Likelihood

Binomial likelihood
```math
p(y = k | f) = k!/(n!(n-k)!) θᵏ(1 - θ)^{n-k}
```
for number of successes ``k ∈ \\{0, 1, …, n\\}`` out of ``n`` Bernoulli trials, where
``θ = \\exp(f)/(1 + \\exp(f))`` and ``f`` is the latent Gaussian process.
"""
struct BinLik <: Likelihood
    "Fixed number of trials"
    n::Int
end

function log_dens(binomial::BinLik, f::AbstractVector, y::Vector{Int})
    θ = @. exp(f) / (1 + exp(f))
    return Float64[lgamma(binomial.n+1.0) - lgamma(yi+1.0) - lgamma(binomial.n-yi+1.0) + yi*log(θi) + (binomial.n-yi)*log(1-θi) for (θi,yi) in zip(θ,y)]
end

function dlog_dens_df(binomial::BinLik, f::AbstractVector, y::Vector{Int})
    return Float64[yi/(1.0+exp(fi)) - (binomial.n-yi)*exp(fi)/(1+exp(fi)) for (fi,yi) in zip(f,y)]
end

#mean and variance under the likelihood
mean_lik(binomial::BinLik, f::AbstractVector) = Float64[binomial.n*exp(fi)/(1.0+exp(fi)) for fi in f]
var_lik(binomial::BinLik, f::AbstractVector) = Float64[binomial.n*(exp(fi)/(1.0+exp(fi)))*(1.0/(1.0+exp(fi))) for fi in f]

get_params(binomial::BinLik) = []
num_params(binomial::BinLik) = 0


using Pkg
Pkg.add("Latexify")
using Latexify

Pkg.add("Plots")
using Plots
Pkg.add("PyPlot")
pyplot()

# Julia code for linear Stommel problem solution in "F. X. GIRALDO AND M. RESTELLI"
# High-order semi-implicit time-integrators for a triangular discontinuous Galerkin
# oceanic shallow water model

L=1e6 # Basin size
τ₀=0.2
β=1e-11
f₀=1e-4
γ=1e-6
g=10
H=1000
ρ=1000

# Derived terms
y_m=L/2
ϕ_B=g*H

ETMP01=:( ((β/γ)^2+4*(π/L)^2)^0.5 )
λ₁_def=:( λ₁ = (-β/γ+$ETMP01)/2 )
λ₂_def=:( λ₂ = (-β/γ-$ETMP01)/2 )
display( latexify(λ₁_def) )
display( latexify(λ₂_def) )

C₀_def=:( C₀ = τ₀/(γ*ρ*H)*(L/π) )
C₁_def=:( C₁ =  C₀*( (1-exp(λ₂*L))  / (exp(λ₂*L) - exp(λ₁*L)) ) )
C₂_def=:( C₂ = -C₀*( (1-exp(λ₁*L))  / (exp(λ₂*L) - exp(λ₁*L)) ) )
display( latexify(C₀_def) )
display( latexify(C₁_def) )
display( latexify(C₂_def) )

ψ_def=:( ψ = (C₀ + C₁*exp(λ₁*x) + C₂*exp(λ₂*x))*sin((π*y)/L) )
display( latexify(ψ_def) )

display( λ₁_def )
display( λ₂_def )
display( C₀_def )
display( C₁_def )
display( C₂_def )
display( ψ_def  )

λ₁ = (-β / γ + ((β / γ) ^ 2 + 4 * (π / L) ^ 2) ^ 0.5) / 2
λ₂ = (-β / γ - ((β / γ) ^ 2 + 4 * (π / L) ^ 2) ^ 0.5) / 2
C₀ = (τ₀ / (γ * ρ * H)) * (L / π)
C₁ = C₀ * ((1 - exp(λ₂ * L)) / (exp(λ₂ * L) - exp(λ₁ * L)))
C₂ = -C₀ * ((1 - exp(λ₁ * L)) / (exp(λ₂ * L) - exp(λ₁ * L)))
ψ(x,y) = (C₀ + C₁ * exp(λ₁ * x) + C₂ * exp(λ₂ * x)) * sin((π * y) / L)

nx=ny=50;
x=collect(range(0.,L,step=L/nx))
y=collect(range(0.,L,step=L/ny))
ψval=zeros(nx+1,ny+1)
for j=1:ny
 for i=1:nx
  ψval[i,j]=ψ(x[i],y[j])
 end
end
ψval

p1 = contour(x, y, ψval')
plot(p1)

ETMP01=:( C₀*β*L/π*cos(π*y/L) +fofy*ψ )
ETMP02=:( C₁/λ₁*exp(λ₁*x)+C₂/λ₂*exp(λ₂*x) )
display( latexify(ETMP01) )
display( latexify(ETMP02) )
ϕ_s_def=:( ϕ_s = $ETMP01 + γ*π/L*cos(π*y/L)*$ETMP02 )
display( latexify(ϕ_s_def) )

ϕ_s_func(x,y,fofy)=(((C₀ * β * L) / π) * cos((π * y) / L) + fofy * ψval[x,y]) + ((γ * π) / L) * cos((π * y) / L) * ((C₁ / λ₁) * exp(λ₁ * x) + (C₂ / λ₂) * exp(λ₂ * x))

ϕ_s_val=zeros(nx+1,ny+1)
for j=1:ny+1
 for i=1:nx+1
  ϕ_s_val[i,j]=ϕ_s_func( i,j,f₀+β*y[j] )
 end
end

p1 = contour(x, y, ϕ_s_val')
plot(p1)


# Copyright 2017, Iain Dunning, Joey Huchette, Miles Lubin, and contributors    #src
# This Source Code Form is subject to the terms of the Mozilla Public License   #src
# v.2.0. If a copy of the MPL was not distributed with this file, You can       #src
# obtain one at https://mozilla.org/MPL/2.0/.                                   #src

# # The minimum distortion problem

# This example arises from computational geometry, in particular the problem of
# embedding a general finite metric space into a euclidean space.
#
# It is known that the 4-point metric space defined by the star graph:
#
#     x
#      \\
#       x — x
#      /
#     x
#
# where distances are computed by length of the shortest path between vertices,
# cannot be exactly embedded into a euclidean space of any dimension.
#
# Here we will formulate and solve an SDP to compute the best possible embedding,
# that is, the embedding f() that minimizes the distortion c such that
#
#     (1 / c) * D(a, b) ≤ ||f(a) - f(b)|| ≤ D(a, b)
#
# for all points (a, b), where D(a, b) is the distance in the metric space.
#
# Any embedding can be characterized by its Gram matrix Q, which is PSD, and
#
#     ||f(a) - f(b)||^2 = Q[a, a] + Q[b, b] - 2 * Q[a, b]
#
# We can therefore constrain
#
#     D[i, j]^2 ≤ Q[i, i] + Q[j, j] - 2 * Q[i, j] ≤ c^2 * D[i, j]^2
#
# and minimize c^2, which gives us the SDP formulation below.
#
# For more detail, see "Lectures on discrete geometry" by J. Matoušek, Springer,
# 2002, pp. 378-379.

using JuMP
import SCS
import Test

function example_min_distortion()
    model = Model(SCS.Optimizer)
    set_silent(model)
    D = [
        0.0 1.0 1.0 1.0;
        1.0 0.0 2.0 2.0;
        1.0 2.0 0.0 2.0;
        1.0 2.0 2.0 0.0
    ]
    @variable(model, c² >= 1.0)
    @variable(model, Q[1:4, 1:4], PSD)
    for i in 1:4
        for j in (i + 1):4
            @constraint(model, D[i, j]^2 <= Q[i, i] + Q[j, j] - 2 * Q[i, j])
            @constraint(model, Q[i, i] + Q[j, j] - 2 * Q[i, j] <= c² * D[i, j]^2)
        end
    end
    @objective(model, Min, c²)
    optimize!(model)
    Test.@test termination_status(model) == MOI.OPTIMAL
    Test.@test primal_status(model) == MOI.FEASIBLE_POINT
    Test.@test objective_value(model) ≈ 4/3 atol = 1e-4
    return
end

example_min_distortion()


"""
    newton(f,dfdx,x₁[;maxiter,ftol,xtol])

Use Newton's method to find a root of `f` starting from `x₁`, where
`dfdx` is the derivative of `f`. Returns a vector of root estimates.

The optional keyword parameters set the maximum number of iterations
and the stopping tolerance for values of `f` and changes in `x`.
"""
function newton(f,dfdx,x₁;maxiter=40,ftol=100*eps(),xtol=100*eps())
    x = [float(x₁)]
    y = f(x₁)
    Δx = Inf   # for initial pass below
    k = 1

    while (abs(Δx) > xtol) && (abs(y) > ftol)
        dydx = dfdx(x[k])
        Δx = -y/dydx            # Newton step
        push!(x,x[k]+Δx)        # append new estimate

        k += 1
        y = f(x[k])
        if k==maxiter
            @warn "Maximum number of iterations reached."
            break   # exit loop
        end
    end
    return x
end

"""
    secant(f,x₁,x₂[;maxiter,ftol,xtol])

Use the secant method to find a root of `f` starting from `x₁` and
`x₂`. Returns a vector of root estimates.

The optional keyword parameters set the maximum number of iterations
and the stopping tolerance for values of `f` and changes in `x`.
"""
function secant(f,x₁,x₂;maxiter=40,ftol=100*eps(),xtol=100*eps())
    x = [float(x₁),float(x₂)]
    y₁ = f(x₁)
    Δx,y₂ = Inf,Inf   # for initial pass in the loop below
    k = 2

    while (abs(Δx) > xtol) && (abs(y₂) > ftol)
        y₂ = f(x[k])
        Δx = -y₂ * (x[k]-x[k-1]) / (y₂-y₁)   # secant step
        push!(x,x[k]+Δx)        # append new estimate

        k += 1
        y₁ = y₂    # current f-value becomes the old one next time

        if k==maxiter
            @warn "Maximum number of iterations reached."
            break   # exit loop
        end
    end
    return x
end

"""
    newtonsys(f,jac,x₁[;maxiter,ftol,xtol])

Use Newton's method to find a root of a system of equations,
starting from `x₁`. The functions `f` and `jac should return the
residual vector and the Jacobian matrix, respectively. Returns the
history of root estimates as a vector of vectors.

The optional keyword parameters set the maximum number of iterations
and the stopping tolerance for values of `f` and changes in `x`.

"""
function newtonsys(f,jac,x₁;maxiter=40,ftol=1000*eps(),xtol=1000*eps())
    x = [float(x₁)]
    y,J = f(x₁),jac(x₁)
    Δx = Inf   # for initial pass below
    k = 1

    while (norm(Δx) > xtol) && (norm(y) > ftol)
        Δx = -(J\y)             # Newton step
        push!(x,x[k] + Δx)    # append to history
        k += 1
        y,J = f(x[k]),jac(x[k])

        if k==maxiter
            @warn "Maximum number of iterations reached."
            break
        end
    end
    return x
end

"""
    fdjac(f,x₀[,y₀])

Compute a finite-difference approximation of the Jacobian matrix for
`f` at `x₀`, where `y₀`=`f(x₀)` may be given.
"""
function fdjac(f,x₀,y₀=f(x₀))
    δ = sqrt(eps())*max(norm(x₀),1)   # FD step size
    m,n = length(y₀),length(x₀)
    if n==1
        J = (f(x₀+δ) - y₀) / δ
    else
        J = zeros(m,n)
        x = copy(x₀)
        for j in 1:n
            x[j] += δ
            J[:,j] = (f(x) - y₀) / δ
            x[j] -= δ
        end
    end
    return J
end

"""
    levenberg(f,x₁[;maxiter,ftol,xtol])

Use Levenberg's quasi-Newton iteration to find a root of the system
`f`, starting from `x₁`, with `tol` as the stopping tolerance in
both step size and residual norm. Returns the history of root estimates
as a vector of vectors.

The optional keyword parameters set the maximum number of iterations
and the stopping tolerance for values of `f` and changes in `x`.

"""
function levenberg(f,x₁;maxiter=40,ftol=1e-12,xtol=1e-12)
    x = [float(x₁)]
    yₖ = f(x₁)
    k = 1;  s = Inf;
    A = fdjac(f,x[k],yₖ)   # start with FD Jacobian
    jac_is_new = true

    λ = 10;
    while (norm(s) > xtol) && (norm(yₖ) > ftol)
        # Compute the proposed step.
        B = A'*A + λ*I
        z = A'*yₖ
        s = -(B\z)

        x̂ = x[k] + s
        ŷ = f(x̂)

        # Do we accept the result?
        if norm(ŷ) < norm(yₖ)    # accept
            λ = λ/10   # get closer to Newton
            # Broyden update of the Jacobian.
            A += (ŷ-yₖ-A*s)*(s'/(s'*s))
            jac_is_new = false

            push!(x,x̂)
            yₖ = ŷ
            k += 1
        else                       # don't accept
            # Get closer to gradient descent.
            λ = 4λ
            # Re-initialize the Jacobian if it's out of date.
            if !jac_is_new
                A = fdjac(f,x[k],yₖ)
                jac_is_new = true
            end
        end

        if k==maxiter
            @warn "Maximum number of iterations reached."
            break
        end

    end
    return x
end


using DiscreteFiltering
using OrdinaryDiffEq: ODEProblem, solve, QNDF
using LinearAlgebra
using Plots
using SparseArrays

## Filter
h₀ = 0.03
h(x) = h₀ * (1 - 1 / 2 * cos(x))
dh(x) = 0.0 # h₀ / 2 * sin(x)
α(x) = 1 / 3 * dh(x) * h(x)
filter = TopHatFilter(h)

## Domain
a = 0.0
b = 2π
domain = PeriodicIntervalDomain(a, b)
# domain = ClosedIntervalDomain(a, b)

## Discretization
N = 500
M = N
x = discretize(domain, M)
ξ = discretize(domain, N)
Δx = (b - a) / N

Δx^2 / maximum(abs.(α.(x)))

## Time
T = 1.0
t = T

## Plot filter
plot(x, h)
ylims!((0, ylims()[2]))

## Get matrices
C = advection_matrix(domain, N)
D = diffusion_matrix(domain, N)
W = filter_matrix(filter, domain, M, N)
R = reconstruction_matrix(filter, domain, M, N)
A = spdiagm(α.(ξ))

## Exact solutions
u₀(ξ) = sin(ξ) + 0.6cos(5ξ) + 0.04sin(20(ξ - 1))

## Discrete initial conditions
uₕ = u₀.(ξ, 0.0)
plot(ξ, uₕ, label = "Discretized")

## Extension to a non-linear case: Burgers equation
ν = 0.03
duₕ(uₕ, p, t) = -uₕ .* (C * uₕ) + ν * D * uₕ
prob = ODEProblem(duₕ, uₕ, (0, T))
sol = solve(prob, QNDF(), abstol = 1e-6, reltol = 1e-4)

function duₕ_allbar(uₕ_allbar, p, t)
    uₕ = R * uₕ_allbar
    -W * (uₕ .* (C * uₕ)) + ν * W * D * uₕ
end
prob_allbar = ODEProblem(duₕ_allbar, W * uₕ, (0, T))
sol_allbar = solve(prob_allbar, QNDF(), abstol = 1e-6, reltol = 1e-4)

plot(ξ, uₕ, label = "Initial")
plot!(x, W * uₕ, label = "Initial discretized-then-filtered")
#plot!(ξ, sol(t), label = "Discretized")
plot!(x, W * sol(t), label = "W * Discretized")
plot!(x, sol_allbar(t), label = "Discretized-then-filtered")
ylims!(minimum(uₕ), maximum(uₕ))


module ConvexHull

struct Point
    x::Float64
    y::Float64
end

function Base.isless(p::Point, q::Point)
    p.x < q.x || (p.x == q.x && p.y < q.y)
end

function isrightturn(p::Point, q::Point, r::Point)
    (q.x - p.x) * (r.y - p.y) - (q.y - p.y) * (r.x - p.x) < 0
end

function grahamscan(points::Vector{Point})
    sorted = sort(points)
    upperhull = halfhull(sorted)
    lowerhull = halfhull(reverse(sorted))
    [upperhull..., lowerhull[2:end-1]...]
end

function halfhull(points::Vector{Point})
    halfhull = points[1:2]
    for p in points[3:end]
        push!(halfhull, p)
        while length(halfhull) > 2 && !isrightturn(halfhull[end-2:end]...)
            deleteat!(halfhull, length(halfhull) - 1)
        end
    end
    halfhull
end

end # module


using NumericalMethodsforEngineers, IterativeSolvers

a = [16. 4. 8.; 4. 5. -4.; 8. -4. 22.]
b = [4., 2., 5.]

upper = cholesky(a).U
lower = upper'
lower |> display
y = lower \ b
c = upper \ y

tol = 1.0e-10
maxiters = 5
x = [(i=i, cg=cg(a, b; maxiter=i)) for i in 1:maxiters]
x|> display

println("\nSolution Vector:")
x[3].cg |> display
println()

@test round.(x[3].cg; digits=6) == round.(c; digits=6)


module NewtonsMethod

greet() = print("Hello World!")

using LinearAlgebra, ForwardDiff, Statistics

function newtonroot(f, f_prime; x0, tol = 1E-7, maxiter = 1000)
    x_old = x0
    iter = 1
    while normdiff > tol && iter <= maxiter
        x_new = x_old - (f(x_old)/f_prime(x_old))
        normdiff = norm(x_new - x_old)
        x_old = x_new
        iter = iter + 1
    end
    if normdiff > tol
        return nothing
    else
        return x_old
    end

end

function newtonroot(f; x0, tol = 1E-7, maxiter = 1000)
    f_prime(x) = ForwardDiff.derivative(f,x)
    x_old = x0
    normdiff = Inf
    iter = 1
    while normdiff > tol && iter <= maxiter
        x_new = x_old - (f(x_old)/f_prime(x_old))
        normdiff = norm(x_new - x_old)
        x_old = x_new
        iter = iter + 1
    end
    if normdiff > tol
        return nothing
    else
        return x_old
    end
end

export newtonroot
end # module


#=
In the 20×20 grid below, four numbers along a diagonal line have been marked in red.

08 02 22 97 38 15 00 40 00 75 04 05 07 78 52 12 50 77 91 08
49 49 99 40 17 81 18 57 60 87 17 40 98 43 69 48 04 56 62 00
81 49 31 73 55 79 14 29 93 71 40 67 53 88 30 03 49 13 36 65
52 70 95 23 04 60 11 42 69 24 68 56 01 32 56 71 37 02 36 91
22 31 16 71 51 67 63 89 41 92 36 54 22 40 40 28 66 33 13 80
24 47 32 60 99 03 45 02 44 75 33 53 78 36 84 20 35 17 12 50
32 98 81 28 64 23 67 10 26 38 40 67 59 54 70 66 18 38 64 70
67 26 20 68 02 62 12 20 95 63 94 39 63 08 40 91 66 49 94 21
24 55 58 05 66 73 99 26 97 17 78 78 96 83 14 88 34 89 63 72
21 36 23 09 75 00 76 44 20 45 35 14 00 61 33 97 34 31 33 95
78 17 53 28 22 75 31 67 15 94 03 80 04 62 16 14 09 53 56 92
16 39 05 42 96 35 31 47 55 58 88 24 00 17 54 24 36 29 85 57
86 56 00 48 35 71 89 07 05 44 44 37 44 60 21 58 51 54 17 58
19 80 81 68 05 94 47 69 28 73 92 13 86 52 17 77 04 89 55 40
04 52 08 83 97 35 99 16 07 97 57 32 16 26 26 79 33 27 98 66
88 36 68 87 57 62 20 72 03 46 33 67 46 55 12 32 63 93 53 69
04 42 16 73 38 25 39 11 24 94 72 18 08 46 29 32 40 62 76 36
20 69 36 41 72 30 23 88 34 62 99 69 82 67 59 85 74 04 36 16
20 73 35 29 78 31 90 01 74 31 49 71 48 86 81 16 23 57 05 54
01 70 54 71 83 51 54 69 16 92 33 48 61 43 52 01 89 19 67 48
The product of these numbers is 26 × 63 × 78 × 14 = 1788696.

What is the greatest product of four adjacent numbers in the same direction (up, down, left, right, or diagonally) in the 20×20 grid?
=#

using DelimitedFiles

function calc(x, i, j, adjacents)
  ar = Function[]
  push!(ar, (i,j)->i<size(x,1)-adjacents) # right
  push!(ar, (i,j)->j<size(x,2)-adjacents) # down
  push!(ar, (i,j)->j>=adjacents) # up
  push!(ar, (i,j)->i>=adjacents) # left
  push!(ar, (i,j)->i+adjacents<size(x,1) && j+adjacents<size(x,2)) # right diagonal
  push!(ar, (i,j)->i-adjacents>0 && j+adjacents<size(x,2)) # left diagonal
  vectors = Array[[1, 0], [0, 1], [0, -1], [-1, 0], [1, 1], [-1, 1]]
  maxwalk = 1
  for element in 1:length(ar)
    if ar[element](i,j)
      maxwalk = max(maxwalk, walk(x, i, j, vectors[element], adjacents))
    end
  end
  maxwalk
end

function walk(x, i, j, vector, adjacents)
  product = 1
  for a in 1:adjacents
      product *= x[j, i]
      i += vector[1]
      j += vector[2]
  end
  product
end

function product_adjacent_numbers()
  x = readdlm(dirname(@__FILE__()) * "/../Resources/p11.txt", ' ', Int64) # Load matrix from file
  adjacents = 4
  maxproduct = 1
  for i in 1:size(x,1), j in 1:size(x,2)
    maxproduct = max(calc(x, i, j, adjacents), maxproduct)
  end
  maxproduct
end
@time println(product_adjacent_numbers())


@doc raw"""
    soft_threshold(X, γ)

Soft thresholding `X` with threshold `γ` using `sign(X) * max(abs(X) - γ, 0)`. Broadcasting is applied
when necessarily.

# Examples

`soft_threshold(X, 1/λ)` is the solution to the simplified lasso problem:

```math
    \min_d \lvert d \rvert + \frac{\lambda}{2} \lVert d - X \rVert^2_2
```

# References

[1] Goldstein, T., & Osher, S. (2009). The split Bregman method for L1-regularized problems. _SIAM journal on imaging sciences_, 2(2), 323-343.

[2] Wikipedia contributors. (2020, July 15). Lasso (statistics). In _Wikipedia, The Free Encyclopedia_. Retrieved 10:40, August 21, 2020, from https://en.wikipedia.org/w/index.php?title=Lasso_(statistics)&oldid=967820964

"""
soft_threshold(X, γ) = soft_threshold.(X, γ)
soft_threshold(x::T, γ::Number) where T <: Number = sign(x) * max(abs(x) - _maybe_promote(T, γ), zero(T))
# TODO: for cases that x is positve, max(x - γ, 0) is slightly faster

_maybe_promote(::Type{T}, x) where T = convert(T, x)
_maybe_promote(::Type{T1}, x::T2) where {T1 <: Union{Bool,Integer},T2} = promote_type(T1, T2)(x)

"""
    block_matching([f=mse,] img, p; kwargs...)

Given the patch center `p` and image `img`, search the `num_patches` most similar patches to
patch_p. The returned results are `Vector{CartesianIndices}` which stores the positions of each
patch.

# Outputs

* `indices::Vector{<:CartesianIndices}`: each item is the CartesianIndices of the matched patch.

# Arguments

* `f`: a smaller `f(patch, ref_patch)` means these two patches are more similar. The default measure
   is `ImageDistances.mse`.
* `img`: the image that block matching is literating over
* `p`: the patch center

# Parameters

* (Required) `num_patches`: the number of patches that should be returned.
* (Required) `patch_size`
* `search_size = size(img)`: the search window size, by default it searches the whole image.
* `stride = patch_size`: loop step size

# Examples

```jldoctest; setup = :(using TestImages, ImageNoise)
julia> img = testimage("cameraman");

julia> p = CartesianIndex(35, 35);

julia> indices = ReduceNoise.block_matching(img, p; num_patches=5, patch_size=7)
5-element Array{CartesianIndices{2,Tuple{UnitRange{Int64},UnitRange{Int64}}},1}:
[...]

julia> first(indices) == p # p is always the first item
true
```

"""
function block_matching(
        f, img, p::CartesianIndex;
        num_patches,
        patch_size,
        search_window_size=size(img),
        patch_stride=patch_size)::Vector{CartesianIndices{ndims(img)}}
    # Offset is not considered, it might work, it might not work. I don't know yet.
    Base.require_one_based_indexing(img)

    patch_size isa Number && (patch_size = ntuple(_ -> patch_size, ndims(img)))
    patch_stride isa Number && (patch_stride = ntuple(_ -> patch_stride, ndims(img)))
    search_window_size isa Number && (search_window_size = ntuple(_ -> search_window_size, ndims(img)))
    all(isodd.(patch_size)) || throw(ArgumentError("`patch_size = $(patch_size)` should be odd numbers"))

    R = CartesianIndices(img)
    rₚ = CartesianIndex(patch_size .÷ 2) # patch radius
    Base.@boundscheck checkbounds(R, p - rₚ)
    Base.@boundscheck checkbounds(R, p + rₚ)

    Δ = CartesianIndex(patch_stride)
    rₛ = CartesianIndex(search_window_size .÷ 2) # search window radius

    # for simplicity, only consider the patches that do not exceed the image boundary
    # This requires at least Julia 1.6: https://github.com/JuliaLang/julia/pull/37829
    I_first, I_last = first(R) + rₚ, last(R) - rₚ
    Rw = vec(max(I_first, p-rₛ):Δ:min(p+rₛ, I_last))
    length(Rw) <= num_patches && throw(ArgumentError("search window size $(search_window_size) is too small to get enough patches"))

    # patch_p is used repeatly so we need a contiguous memeory layout to get better performance
    # TODO: pre-allocate patch_p?
    patch_p = img[p - rₚ:p + rₚ]
    # patch_q = similar(patch_p)
    dist = map(Rw) do q
        patch_q = @view img[q - rₚ:q + rₚ]
        f(patch_p, patch_q)
    end
    dist_ranks = partialsortperm(dist, 1:num_patches)

    return [q - rₚ:q + rₚ for q in Rw[dist_ranks]]
end
block_matching(img, p::CartesianIndex; kwargs...) = block_matching(ssd, img, p::CartesianIndex; kwargs...)

# Threads helper

function get_num_threads()
    blas = LinearAlgebra.BLAS.vendor()
    # Wrap in a try to catch unsupported blas versions
    try
        if blas == :openblas
            return ccall((:openblas_get_num_threads, Base.libblas_name), Cint, ())
        elseif blas == :openblas64
            return ccall((:openblas_get_num_threads64_, Base.libblas_name), Cint, ())
        elseif blas == :mkl
            return ccall((:MKL_Get_Max_Num_Threads, Base.libblas_name), Cint, ())
        end

        # OSX BLAS looks at an environment variable
        if Sys.isapple()
            return tryparse(Cint, get(ENV, "VECLIB_MAXIMUM_THREADS", "1"))
        end
    catch
    end

    return nothing
end

function with_blas_threads(f, num_threads)
    prev_num_threads = get_num_threads()
    prev_num_threads = isnothing(prev_num_threads) ? 1 : prev_num_threads
    BLAS.set_num_threads(num_threads)
    retval = nothing
    try
        retval = f()
    catch err
        rethrow(err)
    finally
        BLAS.set_num_threads(prev_num_threads)
    end

    return retval
end


using ApproxFun, Base.Test

#The first test checks the solution of the integral equation
# u(x) + \int_{-1}^{+1} \frac{e^{y} u(y)}{\sqrt{1-y^2}} dy = f
# on the interval [-1,1].

x=Fun(identity)
w=1/sqrt(1-x^2)
d=domain(x)

Σ=DefiniteIntegral(d)
bandinds(Σ)

@test domainspace(Σ) ==
    JacobiWeight{Chebyshev{Interval{Float64}},Interval{Float64}}(-0.5,-0.5,Chebyshev())

L=I+Σ[exp(x)*w]
bandinds(L)
usol=sin(2x)
f=L*usol
u=L\f
@test norm(u-usol) <= 10eps()

#The second test checks the solution of the integro-differential equation
# u'(x) + x u(x) + \int_{-2}^{+2} sin(y-x) u(y) \sqrt{4-y^2} dy = f
# on the interval [-2,2], with u(-2) = 1.

x=Fun(identity,[-2.,2.])
w=sqrt(4-x^2)
d=domain(x)

D=Derivative(d)
B=ldirichlet(d)
Σ=DefiniteIntegral(.5,.5,d)

@test domainspace(Σ) ==
    JacobiWeight{Ultraspherical{Int,Interval{Float64}},Interval{Float64}}(.5,.5,Ultraspherical(1,d))

K=LowRankFun((x,y)->sin(y-x)*w(y),Ultraspherical(1,d),domainspace(Σ))

L=D+x+Σ[K]
usol=cospi(20x)
f=L*usol
u=[B;L]\[1.;f]

@test norm(u-usol) ≤ 100eps()

Σ = DefiniteIntegral()

f1=Fun(t->cos(cos(t)),[-π,π])
f=Fun(t->cos(cos(t)),Laurent([-π,π]))

@test_approx_eq sum(f1) Σ*f

f1=Fun(t->cos(cos(t))/t,Laurent(Circle()))
f2=Fun(t->cos(cos(t))/t,Fourier(Circle()))
@test_approx_eq Σ*f1 Σ*f2

f1=Fun(t->cos(cos(t)),Laurent([-π,π]))
f2=Fun(t->cos(cos(t)),Fourier([-π,π]))
@test_approx_eq Σ*f1 Σ*f2

## test over arcs

d=exp(im*Interval(0.1,0.2))
x=Fun(d)
w=1/(sqrt(abs(first(d)-x))*sqrt(abs(last(d)-x)))

@test_approx_eq linesum(w) DefiniteLineIntegral()*w

## Volterra integral equation

d = Interval(0.0,1.0)
V = Volterra(d)
K = LowRankFun((x,y)->sin(y-x),d^2)
L = I-V[K]
f = Fun(exp,d)
@test domainspace(L) == Legendre(d)
@test rangespace(L) == Legendre(d)
@test bandrange(V) == -1:0
u = L\f
@test norm(L*u-f) ≤ 20eps()


using Test, PolynomialBases, FastGaussQuadrature

if !haskey(ENV, "JULIA_PKGEVAL") # sympy is not install on https://github.com/JuliaComputing/NewPkgEval.jl
  import SymPy
  x = SymPy.symbols("x")

  @test 0 == SymPy.simplify( legendre(x, 0) - 1 )
  @test 0 == SymPy.simplify( legendre(x, 1) - x )
  @test 0 == SymPy.simplify( legendre(x, 2) - ( 3x^2 - 1 ) / 2 )
  @test 0 == SymPy.simplify( legendre(x, 3) - ( 5x^3 - 3x) / 2 )
  @test 0 == SymPy.simplify( legendre(x, 4) - ( 35x^4 - 30x^2 + 3 ) / 8 )
  @test 0 == SymPy.simplify( legendre(x, 5) - ( 63x^5 - 70x^3 + 15x ) / 8 )
  @test 0 == SymPy.simplify( legendre(x, 6) - ( 231x^6 - 315x^4 + 105x^2 - 5 ) / 16 )

  @test_skip legendre(x, 4)
end
@inferred legendre(10., 4)

# Gauss Legendre nodes and weights
for p in 0:20
    x1, w1 = gausslegendre(p+1)
    @inferred PolynomialBases.gauss_legendre_nodes_and_weights(p)
    x2, w2 = PolynomialBases.gauss_legendre_nodes_and_weights(p)
    @test x1 ≈ x2 atol=1.e-14
    @test w1 ≈ w2 atol=1.e-14
end

# Lobatto Legendre nodes and weights
for p in 1:20
    x1, w1 = gausslobatto(p+1)
    @inferred PolynomialBases.lobatto_legendre_nodes_and_weights(p)
    x2, w2 = PolynomialBases.lobatto_legendre_nodes_and_weights(p)
    @test x1 ≈ x2 atol=1.e-14
    @test w1 ≈ w2 atol=1.e-14
end

# Derivative and mass matrices for Gauss Legendre nodes and weights
# NOTE: Only some tests for low polynomial degrees since the Vandermonde matrix
#       becomes very ill-conditioned for higher values of p.
for p in 0:8
    basis = GaussLegendre(p)
    V = legendre_vandermonde(basis)

    Dhat = legendre_D(p)
    @test basis.D ≈ V*Dhat/V atol=1.e-13

    Mhat = legendre_M(p)
    @test Diagonal(basis.weights) ≈ V'\Mhat/V atol=1.e-13
end


using SubspacePreservingSparsification
using SparseArrays
using LinearAlgebra
M = [17.05 16.98 0.3 6.99 7; 16.98 0.2 7.1 6.9 0; 0.3 7.1 -12 0.01 17; 6.99 6.9 0.01 -11.97 0; 7 0 17 0 -0.1]

M_id = sparsity_pattern(M, 0.6, 2)

bin_sparse_matrix!(M, M_id, 200)

binned_minimization(M_id, zeros(5, 5), Matrix{Float64}(I, 5, 5), M)

sparsify(M, 0.6, 2, 200)

sparsify(M, 0.6, 2, 200, true)

# This file was generated using Literate.jl, https://github.com/fredrikekre/Literate.jl


using Symbolics, Test
using DomainSets
@variables x y

I1 = Integral(x in ClosedInterval(1, 5))
I2 = Integral(x in ClosedInterval(1, 5))
@test I1 == I2

@variables v(..) u(..) x y
D = Differential(x)
Dxx = Differential(x)^2
I = Integral(y in ClosedInterval(1, 5))
eq = D(I(u(x,y))) ~ 0
eq_test = I(D(u(x,y)))
@test isequal(expand_derivatives(eq.lhs), Symbolics.value(eq_test))

eq = Dxx((I(u(x,y)))) + I(D(u(x,y))) ~ 0
eq_test = I(D(u(x,y))) + I(D(D(u(x,y))))
@test isequal(expand_derivatives(eq.lhs), Symbolics.value(eq_test))

I = Integral(y in ClosedInterval(1, v(x)))
eq = D((I(u(x,y)))) ~ 0
eq_test = I(D(u(x,y))) + D(v(x))*u(x, v(x))
@test isequal(expand_derivatives(eq.lhs), Symbolics.value(eq_test))

I = Integral(y in ClosedInterval(1, v(x)))
eq = Dxx((I(u(x,y)))) ~ 0
eq_test = D(I(D(u(x,y))) + D(v(x))*u(x, v(x)))
eq_test_ = I(D(D(u(x,y)))) + D(D(v(x)))*u(x, v(x)) + 2D(u(x,v(x)))*D(v(x))
@test isequal(expand_derivatives(eq.lhs), Symbolics.value(eq_test_))
@test isequal(expand_derivatives(eq.lhs), expand_derivatives(eq_test))

eq = D((I(u(x,y)^2))) ~ 0
eq_test = I(2D(u(x,y))*u(x,y)) + D(v(x))*u(x, v(x))^2
@test isequal(expand_derivatives(eq.lhs), Symbolics.value(eq_test))


using Distributions, Random
Random.seed!(1)

function queueDES(T, arrF, serF, capacity = Inf, initQ = 0)
    t, q, qL = 0.0, initQ, 0.0

    nextArr, nextSer = arrF(), q == 0 ? Inf : serF()
    while t < T
        tPrev, qPrev = t, q
        if nextSer < nextArr
            t = nextSer
            q -= 1
            if q > 0
                nextSer = t + serF()
            else
                nextSer = Inf
            end
        else
            t = nextArr
            if q == 0
                nextSer = t + serF()
            end
            if q < capacity
                q += 1
            end
            nextArr = t + arrF()
        end
        qL += (t - tPrev)*qPrev
    end
    return qL/t
end

lam, mu, K = 0.82, 1.3, 5
rho = lam/mu
T = 10^6

mm1Theor = rho/(1-rho)
md1Theor = rho/(1-rho)*(2-rho)/2
mm1kTheor = rho/(1-rho)*(1-(K+1)*rho^K+K*rho^(K+1))/(1-rho^(K+1))

mm1Est = queueDES(T,()->rand(Exponential(1/lam)),
            			  ()->rand(Exponential(1/mu)))
md1Est = queueDES(T,()->rand(Exponential(1/lam)),
                                   ()->1/mu)
mm1kEst = queueDES(T,()->rand(Exponential(1/lam)),
                                      ()->rand(Exponential(1/mu)), K)

println("The load on the system: ",rho)
println("Queueing theory: ", (mm1Theor,md1Theor,mm1kTheor) )
println("Via simulation: ", (mm1Est,md1Est,mm1kEst) )