versioninfo()

# If needed:
#Pkg.add("DifferentialEquations")
#Pkg.add("PyPlot")
#Pkg.add("Plots")

using Plots
gr()
using DifferentialEquations

b = 0.25
c = 5.0

y0 = [pi - 0.1; 0.0]

function pend(t, y, dy)
    dy[1] = y[2]
    dy[2] = (-b * y[2]) - (c * sin(y[1]))
end

function f_pend(y, t)
    return [y[2], (-b * y[2]) - (c * sin(y[1]))]
end

tspan = (0.0, 10.0)

function odeint_1(f, y0, tspan)
    prob = ODEProblem(f, y0, tspan)
    sol = solve(prob)
    return sol.t, hcat(sol.u...)'
end

function odeint(f, y0, tspan)
    t, sol = odeint_1(f, y0, tspan)
    return sol
end

t, sol = odeint_1(pend, y0, tspan)

plot(t, sol[:, 1], xaxis="Time t", title="Solution to the pendulum ODE", label="\\theta (t)")
plot!(t, sol[:, 2], label="\\omega (t)")

function rungekutta1(f, y0, t)
    n = length(t)
    y = zeros((n, length(y0)))
    y[1,:] = y0
    for i in 1:n-1
        h = t[i+1] - t[i]
        y[i+1,:] = y[i,:] + h * f(y[i,:], t[i])
    end
    return y
end

t = linspace(0, 10, 101);
sol = rungekutta1(f_pend, y0, t);

plot(t, sol[:, 1], xaxis="Time t", title="Solution to the pendulum ODE with Runge-Kutta 1", label="\\theta (t)")
plot!(t, sol[:, 2], label="\\omega (t)")

t2 = linspace(0, 10, 1001);
sol2 = rungekutta1(f_pend, y0, t2);

plot(t2, sol2[:, 1], xaxis="Time t", title="Solution to the pendulum ODE with Runge-Kutta 1", label="\\theta (t)")
plot!(t2, sol2[:, 2], label="\\omega (t)")

t3 = linspace(0, 10, 2001);
sol3 = rungekutta1(f_pend, y0, t3);

plot(t3, sol3[:, 1], xaxis="Time t", title="Solution to the pendulum ODE with Runge-Kutta 1", label="\\theta (t)")
plot!(t3, sol3[:, 2], label="\\omega (t)")

function rungekutta2(f, y0, t)
    n = length(t)
    y = zeros((n, length(y0)))
    y[1,:] = y0
    for i in 1:n-1
        h = t[i+1] - t[i]
        y[i+1,:] = y[i,:] + h * f(y[i,:] + f(y[i,:], t[i]) * h/2, t[i] + h/2)
    end
    return y
end

t3 = linspace(0, 10, 21);
sol3 = rungekutta2(f_pend, y0, t3);

plot(t3, sol3[:, 1], xaxis="Time t", title="Solution to the pendulum ODE with Runge-Kutta 2 (21 points)", label="\\theta (t)")
plot!(t3, sol3[:, 2], label="\\omega (t)")

t3 = linspace(0, 10, 101);
sol3 = rungekutta2(f_pend, y0, t3);

plot(t3, sol3[:, 1], xaxis="Time t", title="Solution to the pendulum ODE with Runge-Kutta 2 (101 points)", label="\\theta (t)")
plot!(t3, sol3[:, 2], label="\\omega (t)")

function rungekutta4(f, y0, t)
    n = length(t)
    y = zeros((n, length(y0)))
    y[1,:] = y0
    for i in 1:n-1
        h = t[i+1] - t[i]
        k1 = f(y[i,:], t[i])
        k2 = f(y[i,:] + k1 * h/2, t[i] + h/2)
        k3 = f(y[i,:] + k2 * h/2, t[i] + h/2)
        k4 = f(y[i,:] + k3 * h, t[i] + h)
        y[i+1,:] = y[i,:] + (h/6) * (k1 + 2*k2 + 2*k3 + k4)
    end
    return y
end

t = linspace(0, 10, 31);
sol = rungekutta4(f_pend, y0, t);

plot(t, sol[:, 1], xaxis="Time t", title="Solution to the pendulum ODE with Runge-Kutta 4 (31 points)", label="\\theta (t)")
plot!(t, sol[:, 2], label="\\omega (t)")

t = linspace(0, 10, 101);
sol = rungekutta4(f_pend, y0, t);

plot(t, sol[:, 1], xaxis="Time t", title="Solution to the pendulum ODE with Runge-Kutta 4 (101 points)", label="\\theta (t)")
plot!(t, sol[:, 2], label="\\omega (t)")

methods = [rungekutta1, rungekutta2, rungekutta4]
markers = [:o, :s, :>]

function test_1(n=101)
    t = linspace(0, 10, n)
    tspan = (0.0, 10.0)
    t1, sol = odeint_1(pend, y0, tspan)
    plt = plot(t1, sol[:, 1], marker=:d, xaxis="Time t", title="Solution to the pendulum ODE with ($n points)", label="odeint")
    for (method, m) in zip(methods, markers)
        sol = method(f_pend, y0, t)
        plot!(t, sol[:, 1], marker=m, label=string(method))
    end
    display(plt)
end

test_1(10)

test_1(20)

test_1(100)

test_1(200)

y0 = [0; 1; 0.1]

function f(y, t)
    return [y[2], y[3], 12 * y[1]^(4/5) + cos(y[2])^3 - sin(y[3])]
end

function f_2(t, y, dy)
    dy[1] = y[2]
    dy[2] = y[3]
    dy[3] = 12 * y[1]^(4/5) + cos(y[2])^3 - sin(y[3])
end

function test_2(n=101)
    t = linspace(0, 1, n)
    tspan = (0.0, 1.0)
    t1, sol = odeint_1(f_2, y0, tspan)
    plt = plot(t1, sol[:, 1], marker=:d, xaxis="Time t", title="Solution to an ODE with ($n points)", label="odeint")
    for (method, m) in zip(methods, markers)
        sol = method(f, y0, t)
        plot!(t, sol[:, 1], marker=m, label=string(method))
    end
    display(plt)
end

test_2(10)

test_2(50)

y0 = [10.0, -3.0, 1.0, 1.0]

function f(y, t)
    return [y[2], y[3], y[4], y[1]^(-5/3)]
end

function f_2(t, y, dy)
    dy[1] = y[2]
    dy[2] = y[3]
    dy[3] = y[4]
    dy[4] = y[1]^(-5/3)
end

function test_3(n=101)
    t = linspace(0, 1, n)
    tspan = (0.0, 1.0)
    t1, sol = odeint_1(f_2, y0, tspan)
    plt = plot(t1, sol[:, 1], marker=:d, xaxis="Time t", title="Solution to an ODE with ($n points)", label="odeint")
    for (method, m) in zip(methods, markers)
        sol = method(f, y0, t)
        plot!(t, sol[:, 1], marker=m, label=string(method))
    end
    display(plt)
end

test_3(10)

test_3(50)

function benchmark(n=101)
    t = linspace(0, 1, n)
    tspan = (0.0, 1.0)

    print("Time of solving an ODE with the 'solve' method from 'DifferentialEquations' ...\n")
    @time t1, sol = odeint_1(f_2, y0, tspan)
    for method in methods
        print("Time of solving an ODE with the $method method for $n points ...\n")
        @time sol = method(f, y0, t)
    end
end

for n in [20, 100, 1000]
    print("\nFor $n points...\n")
    benchmark(n)
end

using BenchmarkTools, Compat

function benchmark(n=101)
    t = linspace(0, 1, n)
    tspan = (0.0, 1.0)

    print("Time of solving an ODE with the 'solve' method from 'DifferentialEquations' ...\n")
    @btime t1, sol = $odeint_1($f_2, $y0, $tspan)
    for method in methods
        print("Time of solving an ODE with the $method method for $n points ...\n")
        @btime sol = $method($f, $y0, $t)
    end
end

for n in [20, 100, 1000]
    print("\nFor $n points...\n")
    benchmark(n)
end


"""
    legendre(x, p::Integer)

Evaluate the Legendre polynomial of degree `p` at `x` using the three term
recursion [Kopriva, Implementing Spectral Methods for PDEs, Algorithm 20].
"""
function legendre(x, p::Integer)
    a::typeof(x) = one(x)
    b::typeof(x) = x

    if p <= 0
        return a
    elseif p == 1
        return b
    end

    for j in 2:p
        a, b = b, ( (2j-1)*x*b - (j-1)*a ) / j
    end

    b
end

"""
    legendre_and_derivative(x, p::Integer)

Evaluate the Legendre polynomial of degree `p` and its derivative at `x` using
the three term recursion [Kopriva, Implementing Spectral Methods for PDEs,
Algorithm 22].
"""
function legendre_and_derivative(x, p::Integer)
    # coefficients for the polynomial...
    a::typeof(x) = one(x)
    b::typeof(x) = x
    # ... and for the derivative
    aa::typeof(x) = zero(x)
    bb::typeof(x) = one(x)

    if p <= 0
        return a, aa
    elseif p == 1
        return b, bb
    end

    for j in 2:p
        a, b = b, ( (2j-1)*x*b - (j-1)*a ) / j
        aa, bb = bb, aa + (2j-1)*a
    end

    b, bb
end

"""
    gauss_legendre_nodes_and_weights(p, T=Float64::Type, tol=4*eps(T), maxit=100)

Compute the Gauss-Legendre nodes and weights for polynomials of degree `p`
using the scalar type `T`, tolerance `tol` and maximal number of Newton iterations
`maxit` [Kopriva, Implementing Spectral Methods for PDEs, Algorithm 23].
"""
function gauss_legendre_nodes_and_weights(p, T=Float64::Type, tol=4*eps(T), maxit=100)
    if p <= 0
        return T[0], T[2]
    elseif p == 1
        x0 = -1 / sqrt(T(3))
        return [x0, -x0], T[1, 1]
    end

    nodes = Vector{T}(undef, p+1)
    weights = Vector{T}(undef, p+1)
    for j in 0:((p+1)÷2-1)
        x = -cospi(T(2j+1)/(2p+2))
        for k in 1:maxit
            pol, der = legendre_and_derivative(x, p+1)
            Δ = -pol / der
            x = x + Δ
            abs(Δ) <= tol*abs(x) && break
        end
        pol, der = legendre_and_derivative(x, p+1)
        nodes[j+1] = x
        nodes[p-j+1] = -x
        weights[j+1] = weights[p-j+1] = 2 / ( (1-x^2)*der^2 )
    end

    if mod(p,2) == 0
        pol, der = legendre_and_derivative(zero(T), p+1)
        nodes[p÷2+1]   = 0
        weights[p÷2+1] = 2 / der^2
    end

    return nodes, weights
end

# helper function [Kopriva, Implementing Spectral Methods for PDEs, Algorithm 24]
function q_and_L_evaluation(x, p::Integer)
    a = one(x)
    b = x
    aa = zero(x)
    bb = one(x)

    for j in 2:p
        a, b = b, ( (2j-1)*x*b - (j-1)*a ) / j
        aa, bb = bb, aa + (2j-1)*a
    end
    pol = ( (2p+1)*x*b - p*a ) / (p+1)
    der = aa + (2p+1)*b

    pol-a, der-aa, b
end

"""
    lobatto_legendre_nodes_and_weights(p, T=Float64::Type, tol=4*eps(T), maxit=100)

Compute the Lobatto-Legendre nodes and weights for polynomials of degree `p`
using the scalar type `T`, tolerance `tol` and maximal number of Newton iterations
`maxit` [Kopriva, Implementing Spectral Methods for PDEs, Algorithm 25].
"""
function lobatto_legendre_nodes_and_weights(p, T=Float64::Type, tol=4*eps(T), maxit=100)
    if p <= 0
        return T[0], T[2]
    elseif p == 1
        return T[-1, 1], T[1, 1]
    end

    nodes = Vector{T}(undef, p+1)
    weights = Vector{T}(undef, p+1)

    nodes[1] = -1
    nodes[end] = 1
    weights[1] = weights[end] = T(2) / (p*(p+1))

    for j in 1:((p+1)÷2-1)
        x = -cos( T(j+1//4)*π/p - 3/(8p*T(j+1//4)*π) )
        for k in 1:maxit
            pol, der, leg = q_and_L_evaluation(x, p)
            Δ = -pol / der
            x = x + Δ
            abs(Δ) <= tol*abs(x) && break
        end
        pol, der, leg = q_and_L_evaluation(x, p)
        nodes[j+1] = x
        nodes[p-j+1] = -x
        weights[j+1] = weights[p-j+1] = 2 / ( p*(p+1)*leg^2 )
    end

    if mod(p,2) == 0
        pol, der, leg = q_and_L_evaluation(zero(T), p)
        nodes[p÷2+1]   = 0
        weights[p÷2+1] = 2 / ( p*(p+1)*leg^2 )
    end

    return nodes, weights
end

"""
    legendre_vandermonde(nodes)

Computes the Vandermonde matrix with respect to the Legendre polynomials and
the nodal basis on `nodes`.
The Vandermonde matrix `V` is the transformation matrix from the modal Legendre
basis to the nodal Lagrange basis associated with `nodes`.
"""
function legendre_vandermonde(nodes::AbstractVector)
    T = eltype(nodes)
    pp1 = length(nodes)
    V = Array{T}(undef, pp1, pp1)
    for j in 1:pp1, (i,x) in enumerate(nodes)
      V[i, j] = legendre(x, j-1)
    end
    V
end

legendre_vandermonde(basis::NodalBasis{Line}) = legendre_vandermonde(basis.nodes)

"""
    legendre_M(p, T=Float64)

Computes the diagonal mass matrix in the modal Legendre basis up to degree `p`
using the scalar type `T`.
"""
function legendre_M(p, T=Float64)
  Diagonal( T[2//(2n+1) for n in 0:p] )
end

"""
    legendre_D(p, T=Float64)

Computes the derivative matrix in the modal Legendre basis up to degree `p`
using the scalar type `T`.
"""
function legendre_D(p, T=Float64)
  D = fill(zero(T), p+1, p+1)
  if p >= 1
    D[1, 2] = 1
  end
  if p >= 2
    D[2, 3] = 3
  end
  for col in 4:p+1
    D[:, col] = D[:, col-2]
    D[col-1, col] = 2*col-3
  end
  D
end


function factorial(n::Integer)
    if n < 0
        return zero(n)
    end
    f = one(n)
    for i = 2:n
        f *= i
    end
    return f
end

# computes n!/k!
function factorial{T<:Integer}(n::T, k::T)
    if k < 0 || n < 0 || k > n
        return zero(T)
    end
    f = one(T)
    while n > k
        f *= n
        n -= 1
    end
    return f
end

function binomial{T<:Integer}(n::T, k::T)
    k < 0 && return zero(T)
    sgn = one(T)
    if n < 0
        n = -n + k -1
        if isodd(k)
            sgn = -sgn
        end
    end
    k > n && return zero(T)
    (k == 0 || k == n) && return sgn
    k == 1 && return sgn*n
    if k > (n>>1)
        k = (n - k)
    end
    x::T = nn = n - k + 1
    nn += 1
    rr = 2
    while rr <= k
        xt = div(widemul(x, nn), rr)
        x = xt
        x == xt || throw(OverflowError())
        rr += 1
        nn += 1
    end
    convert(T, copysign(x, sgn))
end

## other ordering related functions ##

function shuffle!(a::AbstractVector)
    for i = length(a):-1:2
        j = rand(1:i)
        a[i], a[j] = a[j], a[i]
    end
    return a
end

shuffle(a::AbstractVector) = shuffle!(copy(a))

function randperm(n::Integer)
    a = Array(typeof(n), n)
    a[1] = 1
    for i = 2:n
        j = rand(1:i)
        a[i] = a[j]
        a[j] = i
    end
    return a
end

function randcycle(n::Integer)
    a = Array(typeof(n), n)
    a[1] = 1
    for i = 2:n
        j = rand(1:i-1)
        a[i] = a[j]
        a[j] = i
    end
    return a
end

function nthperm!(a::AbstractVector, k::Integer)
    k -= 1 # make k 1-indexed
    n = length(a)
    f = factorial(oftype(k, n-1))
    for i=1:n-1
        j = div(k, f) + 1
        k = k % f
        f = div(f, n-i)

        j = j+i-1
        elt = a[j]
        for d = j:-1:i+1
            a[d] = a[d-1]
        end
        a[i] = elt
    end
    a
end
nthperm(a::AbstractVector, k::Integer) = nthperm!(copy(a),k)

function nthperm{T<:Integer}(p::AbstractVector{T})
    isperm(p) || error("argument is not a permutation")
    k, n = 1, length(p)
    for i = 1:n-1
        f = factorial(n-i)
        for j = i+1:n
            k += ifelse(p[j] < p[i], f, 0)
        end
    end
    return k
end

function invperm(a::AbstractVector)
    b = zero(a) # similar vector of zeros
    n = length(a)
    for i = 1:n
        j = a[i]
        ((1 <= j <= n) && b[j] == 0) ||
            error("argument is not a permutation")
        b[j] = i
    end
    b
end

function isperm(A)
    n = length(A)
    used = falses(n)
    for a in A
        (0 < a <= n) && (used[a] $= true) || return false
    end
    true
end

function permute!!{T<:Integer}(a, p::AbstractVector{T})
    count = 0
    start = 0
    while count < length(a)
        ptr = start = findnext(p, start+1)
        temp = a[start]
        next = p[start]
        count += 1
        while next != start
            a[ptr] = a[next]
            p[ptr] = 0
            ptr = next
            next = p[next]
            count += 1
        end
        a[ptr] = temp
        p[ptr] = 0
    end
    a
end

permute!(a, p::AbstractVector) = permute!!(a, copy(p))

function ipermute!!{T<:Integer}(a, p::AbstractVector{T})
    count = 0
    start = 0
    while count < length(a)
        start = findnext(p, start+1)
        temp = a[start]
        next = p[start]
        count += 1
        while next != start
            temp_next = a[next]
            a[next] = temp
            temp = temp_next
            ptr = p[next]
            p[next] = 0
            next = ptr
            count += 1
        end
        a[next] = temp
        p[next] = 0
    end
    a
end

ipermute!(a, p::AbstractVector) = ipermute!!(a, copy(p))

immutable Combinations{T}
    a::T
    t::Int
end

eltype(c::Combinations) = typeof(c.a)
eltype{T}(c::Combinations{UnitRange{T}}) = Array{T,1}
eltype{T}(c::Combinations{Range{T}}) = Array{T,1}

length(c::Combinations) = binomial(length(c.a),c.t)

function combinations(a, t::Integer)
    if t < 0
        # generate 0 combinations for negative argument
        t = length(a)+1
    end
    Combinations(a, t)
end

start(c::Combinations) = [1:c.t]
function next(c::Combinations, s)
    comb = c.a[s]
    if c.t == 0
        # special case to generate 1 result for t==0
        return (comb,[length(c.a)+2])
    end
    s = copy(s)
    for i = length(s):-1:1
        s[i] += 1
        if s[i] > (length(c.a) - (length(s)-i))
            continue
        end
        for j = i+1:endof(s)
            s[j] = s[j-1]+1
        end
        break
    end
    (comb,s)
end
done(c::Combinations, s) = !isempty(s) && s[1] > length(c.a)-c.t+1

immutable Permutations{T}
    a::T
end

eltype(c::Permutations) = typeof(c.a)
eltype{T}(c::Permutations{UnitRange{T}}) = Array{T,1}
eltype{T}(c::Permutations{Range{T}}) = Array{T,1}

length(c::Permutations) = factorial(length(c.a))

permutations(a) = Permutations(a)

start(p::Permutations) = [1:length(p.a)]
function next(p::Permutations, s)
    if length(p.a) == 0
        # special case to generate 1 result for len==0
        return (p.a,[1])
    end
    s = copy(s)
    perm = p.a[s]
    k = length(s)-1
    while k > 0 && s[k] > s[k+1];  k -= 1;  end
    if k == 0
        s[1] = length(s)+1   # done
    else
        l = length(s)
        while s[k] >= s[l];  l -= 1;  end
        s[k],s[l] = s[l],s[k]
        reverse!(s,k+1)
    end
    (perm,s)
end
done(p::Permutations, s) = !isempty(s) && s[1] > length(p.a)

# Integer Partitions

immutable IntegerPartitions
    n::Int
end

length(p::IntegerPartitions) = npartitions(p.n)

partitions(n::Integer) = IntegerPartitions(n)

start(p::IntegerPartitions) = Int[]
done(p::IntegerPartitions, xs) = length(xs) == p.n
next(p::IntegerPartitions, xs) = (xs = nextpartition(p.n,xs); (xs,xs))

function nextpartition(n, as)
    if isempty(as);  return Int[n];  end

    xs = similar(as,0)
    sizehint(xs,length(as)+1)

    for i = 1:length(as)-1
        if as[i+1] == 1
            x = as[i]-1
            push!(xs, x)
            n -= x
            while n > x
                push!(xs, x)
                n -= x
            end
            push!(xs, n)

            return xs
        end
        push!(xs, as[i])
        n -= as[i]
    end
    push!(xs, as[end]-1)
    push!(xs, 1)

    xs
end

let _npartitions = (Int=>Int)[]
    global npartitions
    function npartitions(n::Int)
        if n < 0
            0
        elseif n < 2
            1
        elseif (np = get(_npartitions, n, 0)) > 0
            np
        else
            np = 0
            sgn = 1
            for k = 1:n
                np += sgn * (npartitions(n-k*(3k-1)>>1) + npartitions(n-k*(3k+1)>>1))
                sgn = -sgn
            end
            _npartitions[n] = np
        end
    end
end

# Algorithm H from TAoCP 7.2.1.4
# Partition n into m parts
# in colex order (lexicographic by reflected sequence)

immutable FixedPartitions
    n::Int
    m::Int
end

length(f::FixedPartitions) = npartitions(f.n,f.m)

partitions(n::Integer, m::Integer) = (@assert 2 <= m <= n; FixedPartitions(n,m))

start(f::FixedPartitions) = Int[]
done(f::FixedPartitions, s::Vector{Int}) = !isempty(s) && s[1]-1 <= s[end]
next(f::FixedPartitions, s::Vector{Int}) = (xs = nextfixedpartition(f.n,f.m,s); (xs,xs))

function nextfixedpartition(n, m, bs)
    as = copy(bs)
    if isempty(as)
        # First iteration
        as = [n-m+1, ones(Int, m-1)]
    elseif as[2] < as[1]-1
        # Most common iteration
        as[1] -= 1
        as[2] += 1
    else
        # Iterate
        local j
        s = as[1]+as[2]-1
        for j = 3:m
            if as[j] < as[1]-1; break; end
            s += as[j]
        end
        x = as[j] += 1
        for k = j-1:-1:2
            as[k] = x
            s -= x
        end
        as[1] = s
    end

    return as
end

let _nipartitions = ((Int,Int)=>Int)[]
    global npartitions
    function npartitions(n::Int,m::Int)
        if n < m || m == 0
            0
        elseif n == m
            1
        elseif (np = get(_nipartitions, (n,m), 0)) > 0
            np
        else
            _nipartitions[(n,m)] = npartitions(n-1,m-1) + npartitions(n-m,m)
        end
    end
end

# Algorithm H from TAoCP 7.2.1.5
# Set partitions

immutable SetPartitions{T<:AbstractVector}
    s::T
end

length(p::SetPartitions) = nsetpartitions(length(p.s))

partitions(s::AbstractVector) = SetPartitions(s)

start(p::SetPartitions) = (n = length(p.s); (zeros(Int32, n), ones(Int32, n-1), n, 1))
done(p::SetPartitions, s) = !isempty(s) && s[1][1] > 0
next(p::SetPartitions, s) = nextsetpartition(p.s, s...)

function nextsetpartition(s::AbstractVector, a, b, n, m)
    function makeparts(s, a, m)
        temp = [ similar(s,0) for k = 0:m ]
        for i = 1:n
            push!(temp[a[i]+1], s[i])
        end
        filter!(x->!isempty(x), temp)
    end

    if isempty(s);  return ({s}, ([1], Int[], n, 1));  end

    part = makeparts(s,a,m)

    if a[end] != m
        a[end] += 1
    else
        local j
        for j = n-1:-1:1
            if a[j] != b[j]
                break
            end
        end
        a[j] += 1
        m = b[j] + (a[j] == b[j])
        for k = j+1:n-1
            a[k] = 0
            b[k] = m
        end
        a[end] = 0
    end

    return (part, (a,b,n,m))

end

let _nsetpartitions = (Int=>Int)[]
    global nsetpartitions
    function nsetpartitions(n::Int)
        if n < 0
            0
        elseif n < 2
            1
        elseif (wn = get(_nsetpartitions, n, 0)) > 0
            wn
        else
            wn = 0
            for k = 0:n-1
                wn += binomial(n-1,k)*nsetpartitions(n-1-k)
            end
            _nsetpartitions[n] = wn
        end
    end
end

immutable FixedSetPartitions{T<:AbstractVector}
    s::T
    m::Int
end

length(p::FixedSetPartitions) = nfixedsetpartitions(length(p.s),p.m)

partitions(s::AbstractVector,m::Int) = (@assert 2 <= m <= length(s); FixedSetPartitions(s,m))

start(p::FixedSetPartitions) = (n = length(p.s);m=p.m; (vcat(ones(Int, n-m),1:m), vcat(1,n-m+2:n), n))
# state consists of vector a of length n describing to which partition every element of s belongs and
# a vector b of length m describing the first index b[i] that belongs to partition i

done(p::FixedSetPartitions, s) = !isempty(s) && s[1][1] > 1
next(p::FixedSetPartitions, s) = nextfixedsetpartition(p.s,p.m, s...)

function nextfixedsetpartition(s::AbstractVector, m, a, b, n)
    function makeparts(s, a)
        part = [ similar(s,0) for k = 1:m ]
        for i = 1:n
            push!(part[a[i]], s[i])
        end
        return part
    end

    part = makeparts(s,a)

    if a[end] != m
        a[end] += 1
    else
        local j, k
        for j = n-1:-1:1
            if a[j]<m && b[a[j]+1]<j
                break
            end
        end
        if j>1
            a[j]+=1
            for p=j+1:n
                if b[a[p]]!=p
                    a[p]=1
                end
            end
        else
            for k=m:-1:2
                if b[k-1]<b[k]-1
                    break
                end
            end
            b[k]=b[k]-1
            b[k+1:m]=n-m+k+1:n
            a[1:n]=1
            a[b]=1:m
        end
    end

    return (part, (a,b,n))
end

function nfixedsetpartitions(n::Int,m::Int)
    numpart=0
    for k=0:m
        numpart+=(-1)^(m-k)*binomial(m,k)*(k^n)
    end
    numpart=div(numpart,factorial(m))
    return numpart
end

# For a list of integers i1, i2, i3, find the smallest
#     i1^n1 * i2^n2 * i3^n3 >= x
# for integer n1, n2, n3
function nextprod(a::Vector{Int}, x)
    if x > typemax(Int)
        error("unsafe for x bigger than typemax(Int)")
    end
    k = length(a)
    v = ones(Int, k)                  # current value of each counter
    mx = [nextpow(ai,x) for ai in a]  # maximum value of each counter
    v[1] = mx[1]                      # start at first case that is >= x
    p::widen(Int) = mx[1]             # initial value of product in this case
    best = p
    icarry = 1

    while v[end] < mx[end]
        if p >= x
            best = p < best ? p : best  # keep the best found yet
            carrytest = true
            while carrytest
                p = div(p, v[icarry])
                v[icarry] = 1
                icarry += 1
                p *= a[icarry]
                v[icarry] *= a[icarry]
                carrytest = v[icarry] > mx[icarry] && icarry < k
            end
            if p < x
                icarry = 1
            end
        else
            while p < x
                p *= a[1]
                v[1] *= a[1]
            end
        end
    end
    best = mx[end] < best ? mx[end] : best
    return int(best)  # could overflow, but best to have predictable return type
end

# For a list of integers i1, i2, i3, find the largest
#     i1^n1 * i2^n2 * i3^n3 <= x
# for integer n1, n2, n3
function prevprod(a::Vector{Int}, x)
    if x > typemax(Int)
        error("unsafe for x bigger than typemax(Int)")
    end
    k = length(a)
    v = ones(Int, k)                  # current value of each counter
    mx = [nextpow(ai,x) for ai in a]  # allow each counter to exceed p (sentinel)
    first = int(prevpow(a[1], x))     # start at best case in first factor
    v[1] = first
    p::widen(Int) = first
    best = p
    icarry = 1

    while v[end] < mx[end]
        while p <= x
            best = p > best ? p : best
            p *= a[1]
            v[1] *= a[1]
        end
        if p > x
            carrytest = true
            while carrytest
                p = div(p, v[icarry])
                v[icarry] = 1
                icarry += 1
                p *= a[icarry]
                v[icarry] *= a[icarry]
                carrytest = v[icarry] > mx[icarry] && icarry < k
            end
            if p <= x
                icarry = 1
            end
        end
    end
    best = x >= p > best ? p : best
    return int(best)
end


# least squares penalty

export LeastSquares

### ABSTRACT TYPE

abstract type LeastSquares <: ProximableFunction end

is_smooth(f::LeastSquares) = true
is_quadratic(f::LeastSquares) = true

fun_name(f::LeastSquares) = "Least squares penalty"

### CONSTRUCTORS

"""
**Least squares penalty**

    LeastSquares(A, b, λ=1.0; iterative=false)

For a matrix `A`, a vector `b` and a scalar `λ`, returns the function
```math
f(x) = \\tfrac{\\lambda}{2}\\|Ax - b\\|^2.
```
By default, a direct method (based on Cholesky factorization) is used to evaluate `prox!`.
If `iterative=true`, then `prox!` is evaluated approximately using an iterative method instead.
"""
function LeastSquares(A::M, b::V, lam::R=R(1); iterative=false) where {R <: Real, RC <: RealOrComplex{R}, V <: AbstractArray{RC}, M}
    if iterative == false
        LeastSquaresDirect(A, b, lam)
    else
        LeastSquaresIterative(A, b, lam)
    end
end

infer_shape_of_x(A, ::AbstractVector) = (size(A, 2), )
infer_shape_of_x(A, b::AbstractMatrix) = (size(A, 2), size(b, 2))

### INCLUDE CONCRETE TYPES

include("leastSquaresDirect.jl")
include("leastSquaresIterative.jl")


# # Tomography

#-

# Tomography is the process of reconstructing a density distribution from given
# integrals over sections of the distribution. In our example, we will
# work with tomography on black and white images.
# Suppose $x$ be the vector of $n$ pixel densities, with $x_j$
# denoting how white pixel $j$ is.
# Let $y$ be the vector of $m$ line integrals over the image, with $y_i$
# denoting the integral for line $i$.
# We can define a matrix $A$ to describe the geometry of the lines. Entry
# $A_{ij}$ describes how much of pixel $j$ is intersected by line $i$.
# Assuming our measurements of the line integrals are perfect, we have the relationship that
#
# $$
#   y = Ax
# $$
#
# However, anytime we have measurements, there are usually small errors that occur.
# Therefore it makes sense to try to minimize
#
# $$
#  \|y - Ax\|_2^2.
# $$
#
# This is simply an unconstrained least squares problem; something we can readily solve!

using Convex, ECOS, DelimitedFiles, SparseArrays
aux(str) = joinpath(@__DIR__, "aux_files", str) # path to auxiliary files
line_mat_x = readdlm(aux("tux_sparse_x.txt"))
summary(line_mat_x)

line_mat_y = readdlm(aux("tux_sparse_y.txt"))
summary(line_mat_y)

line_mat_val = readdlm(aux("tux_sparse_val.txt"))
summary(line_mat_val)

line_vals = readdlm(aux("tux_sparse_lines.txt"))
summary(line_vals)

# Form the sparse matrix from the data
# Image is 50 x 50
img_size = 50

# The number of pixels in the image
num_pixels = img_size * img_size

line_mat = spzeros(length(line_vals), num_pixels)

num_vals = length(line_mat_val)

for i in 1:num_vals
  x = Int(line_mat_x[i])
  y = Int(line_mat_y[i])
  line_mat[x + 1, y + 1] = line_mat_val[i]
end

pixel_colors = Variable(num_pixels)
## line_mat * pixel_colors should be close to the line_integral_values
## to reflect that, we minimize a norm
objective = sumsquares(line_mat * pixel_colors - line_vals)
problem = minimize(objective)
solve!(problem, () -> ECOS.Optimizer(verbose=0))

rows = zeros(img_size*img_size)
cols = zeros(img_size*img_size)
for i = 1:img_size
  for j = 1:img_size
    rows[(i-1)*img_size + j] = i
    cols[(i-1)*img_size + j] = img_size + 1 - j
  end
end

# Plot the image using the pixel values obtained!

using Plots
image = reshape(evaluate(pixel_colors), img_size, img_size)
heatmap(image, yflip=true, aspect_ratio=1, colorbar=nothing, color=:grays)


function qp1_test()
    #=
        min x^2 + xy + y^2 + yz + z^2
    s.t.
        x + 2y + 3z >= 4 (c1)
        x +  y      >= 1 (c2)
        x,y \in R
    =#
    model = TestModel{Float64}()

    v = MOI.add_variables(model, 3)

    cf1 =
        MOI.ScalarAffineFunction(MOI.ScalarAffineTerm.([1.0, 2.0, 3.0], v), 0.0)

    c1 = MOI.add_constraint(model, cf1, MOI.GreaterThan(4.0))
    c2 = MOI.add_constraint(
        model,
        MOI.ScalarAffineFunction(
            MOI.ScalarAffineTerm.([1.0, 1.0], [v[1], v[2]]),
            0.0,
        ),
        MOI.GreaterThan(1.0),
    )

    obj = MOI.ScalarQuadraticFunction(
        MOI.ScalarQuadraticTerm.(
            [2.0, 1.0, 2.0, 1.0, 2.0],
            v[[1, 1, 2, 2, 3]],
            v[[1, 2, 2, 3, 3]],
        ),
        MOI.ScalarAffineTerm{Float64}[],
        0.0,
    )
    MOI.set(
        model,
        MOI.ObjectiveFunction{MOI.ScalarQuadraticFunction{Float64}}(),
        obj,
    )
    MOI.set(model, MOI.ObjectiveSense(), MOI.MIN_SENSE)

    return model
end

function qp2_test()
    #=
        min 2 x^2 + y^2 + xy + x + y + 1
    s.t.
        x, y >= 0
        x + y = 1
    =#

    model = TestModel{Float64}()

    x = MOI.add_variable(model)
    y = MOI.add_variable(model)

    MOI.add_constraint(
        model,
        MOI.ScalarAffineFunction(
            MOI.ScalarAffineTerm.([1.0, 1.0], [x, y]),
            0.0,
        ),
        MOI.EqualTo(1.0),
    )

    vc1 = MOI.add_constraint(model, x, MOI.GreaterThan(0.0))
    vc2 = MOI.add_constraint(model, y, MOI.GreaterThan(0.0))

    obj = MOI.ScalarQuadraticFunction(
        MOI.ScalarQuadraticTerm.([4.0, 2.0, 1.0], [x, y, x], [x, y, y]),
        MOI.ScalarAffineTerm.([1.0, 1.0], [x, y]),
        1.0,
    )

    MOI.set(
        model,
        MOI.ObjectiveFunction{MOI.ScalarQuadraticFunction{Float64}}(),
        obj,
    )
    MOI.set(model, MOI.ObjectiveSense(), MOI.MIN_SENSE)

    return model
end


# ------------------------------------------------------------------------------------------
# # Álgebra lineal básica en Julia
# Autor: Andreas Noack Jensen (MIT) (http://www.econ.ku.dk/phdstudent/noack/)
# (con edición de Jane Herriman)
# ------------------------------------------------------------------------------------------

# ------------------------------------------------------------------------------------------
# Julia's syntax is very similar to other languages but there are some important
# differences. Define a matrix of random normal variates
# ------------------------------------------------------------------------------------------

A = rand(1:4,3,3)

# ------------------------------------------------------------------------------------------
# Definir un vector de unos
# ------------------------------------------------------------------------------------------

x = fill(1.0, (3))

# ------------------------------------------------------------------------------------------
# Notamos que $A$ tiene el tipo Array{Int64,2} pero $x$ tiene tipo Array{Int64,1}. Julia
# define los alias a Vector{Type}=Array{Type,1} y Matrix{Type}=Array{Type,2}.
#
# Muchas de las operaciones básicas son idénticas a otros lenguajes
#
# #### Multiplicación
# ------------------------------------------------------------------------------------------

b = A*x

# ------------------------------------------------------------------------------------------
# #### Traspuestas
# Como en otros lenguajes, `A'` es la transpuesta conjugada mientras que `A.'` es sólo la
# traspuesta
# ------------------------------------------------------------------------------------------

Asym = A + A'

# ------------------------------------------------------------------------------------------
# #### Multiplicación traspuesta
# Julia nos permite escribir esto sin *
# ------------------------------------------------------------------------------------------

Apd = A'A

# ------------------------------------------------------------------------------------------
# #### Resolviendo sistemas lineales
# El problema $Ax=b$ para $A$ cuadrada se resulve con la función \.
# ------------------------------------------------------------------------------------------

A\b

# ------------------------------------------------------------------------------------------
# #### Sistemas sobredeterminados
# Cuando nuestra matriz es alta (número de renglones mayores al número de columnas), tenemos
# un sistema lineal sobredeterminado.
#
#
# En este caso \ calcula la de mínimos cuadrados
# ------------------------------------------------------------------------------------------

Atall = rand(3, 2)
display(Atall)
Atall\b

# ------------------------------------------------------------------------------------------
# La función \ también sirve ocn problemas deficientes de rango de mínimos cuadrados. En
# este caso, la solución no es única y Julia regresa el valor con la menor norma.
#
# Para crear un problema de rango deficiente de mínimos cuadrados, vamos a crear una matriz
# deficiente en rango con columnas linealmente dependientes
# ------------------------------------------------------------------------------------------

v = randn(3)
rankdef = [v v]

rankdef\b

# ------------------------------------------------------------------------------------------
# #### Sistemas indeterminados
# cuando A es corta (número de columnas mayor al número de renglones), tenemos un sistema
# indeterminado
#
# En este caso \ regresa la solución con la norma mínima
# ------------------------------------------------------------------------------------------

Ashort = rand(2, 3)
display(Ashort)
Ashort\b[1:2]

# ------------------------------------------------------------------------------------------
# ### Ejercicios
#
#
#
# ```
# A = [
#  1  2  3  4  5  6  7  8  9  10
#  1  2  3  4  5  6  7  8  9  10
#  1  2  3  4  5  6  7  8  9  10
#  1  2  3  4  5  6  7  8  9  10
#  1  2  3  4  5  6  7  8  9  10
#  1  2  3  4  5  6  7  8  9  10
#  1  2  3  4  5  6  7  8  9  10
#  1  2  3  4  5  6  7  8  9  10
#  1  2  3  4  5  6  7  8  9  10
#  1  2  3  4  5  6  7  8  9  10
#  ]
# ```
#
# Quieres obtener
#
# ```
# A = [
#  7  8  9  10  1  2  3  4  5  6
#  7  8  9  10  1  2  3  4  5  6
#  7  8  9  10  1  2  3  4  5  6
#  7  8  9  10  1  2  3  4  5  6
#  7  8  9  10  1  2  3  4  5  6
#  7  8  9  10  1  2  3  4  5  6
#  7  8  9  10  1  2  3  4  5  6
#  7  8  9  10  1  2  3  4  5  6
#  7  8  9  10  1  2  3  4  5  6
#  7  8  9  10  1  2  3  4  5  6
#  ]
# ```
# ------------------------------------------------------------------------------------------

# ------------------------------------------------------------------------------------------
# 10.2 Toma el producto de un vector `v` con sí mismo.
# ------------------------------------------------------------------------------------------

# ------------------------------------------------------------------------------------------
# 10.3 Toma el producto de un vector `v` con sí mismo.
# ------------------------------------------------------------------------------------------

.nbexports/introductory-tutorials/intro-to-julia-ES/10. Algebra lineal basica.jl


using Iterators

function sumSubset(set::Array{Int64,1}, subsetNumber::Int64)
    result = 0
    count = 0
    for i = 0:length(set)-1
        if ((2^i & subsetNumber) > 0)
            count += 1
            result += set[i+1]
        end
    end
    return (result, count)
end

function checkSubsets(set::Array{Int64,1})
    num_subsets = 2^length(set)-1
    for i = 1:num_subsets
        for j = 1:num_subsets
            if (i & j == 0) # treating i,j as bit flags, this is the condition for disjoint
                (sum_i, count_i) = sumSubset(set, i)
                (sum_j, count_j) = sumSubset(set, j)
                #println("i: $count_i, $sum_i; j: $count_j, $sum_j")
                if (sum_i == sum_j)
                    return false
                end
                if (count_i > count_j && sum_i <= sum_j)
                    return false
                end
                if (count_i < count_j && sum_i >= sum_j)
                    return false
                end
            end
        end
    end
    return true
end

# Main
best = nothing
best_sum = Inf

for i in subsets(13:48, 7) # Using rough heuristics -- this takes ~2min to run
    if (checkSubsets(i) && sum(i) < best_sum)
        println("Found one!")
        best_sum = sum(i)
        best = i
    end
end

@show join(best)

#=
Note this is relatively "brute-force". There's a more elegant way to do this which could "learn"
from past failures. e.g. once you know 10 & 15 add to 25, no set with these three numbers should be
considered.
=#


using FFTW
using LinearAlgebra
using DelimitedFiles

# using the convolutional theorem
function convolve_fft(signal1::Array{T}, signal2::Array{T}) where {T <: Number}
    return ifft(fft(signal1).*fft(signal2))
end

function main()

    # sawtooth functions for x and y
    x = [float(i)/200 for i = 1:200]
    y = [float(i)/200 for i = 1:200]

    # Normalization is not strictly necessary, but good practice
    normalize!(x)
    normalize!(y)

    # cyclic convolution via the convolutional theorem
    fft_output = convolve_fft(x, y)

    # outputting convolutions to different files for plotting in external code
    # note: we are outputting just the real component because the imaginary
    #       component is virtually 0
    writedlm("fft.dat", real(fft_output))

end

main()


function get_sensors_ring(; num_pts = 180, φ_inicial = 0, φ_final = 2π, kR = 1, θ = π / 2)
    ϕ_range = range(φ_inicial, φ_final; length = num_pts)
    θ_range = ones(length(ϕ_range)) .* θ

    x = kR .* cos.(ϕ_range) .* sin.(θ_range)
    y = kR .* sin.(ϕ_range) .* sin.(θ_range)
    z = kR .* cos.(θ_range)
    return Array(transpose([x y z]))
end

"""
    get_sensors_sphereSurface_Fibonacci(; num_pts::Integer=100, R=5k₀, θ_lims=(0,π))

Function uses the `Fibonacci Sphere Algorithm` to create `num_pts` points over a sphere shell of radius `R`. This is a Legacy code, prefer the Lebedev points.

Returns each axis (in cartesian coordinates) in row: [x y z]

size([x y z]) = (3,num_pts)
"""
function get_sensors_sphereSurface_Fibonacci(;
    num_pts::Integer = 100,
    R = 5k₀,
    θ_lims = (0, π),
)
    indices = 1:num_pts
    θ_min, θ_max = θ_lims

    θ = acos.(range(cos(θ_min), stop = cos(θ_max), length = num_pts))
    ϕ = π * (1 + sqrt(5)) .* indices

    x, y, z = R .* cos.(ϕ) .* sin.(θ), R .* sin.(ϕ) .* sin.(θ), R .* cos.(θ)
    return Array(transpose([x y z]))
end

"""
    get_sensors_sphereSurface(; R=100k₀, leb_order=125)

return xyz, w

---

Returns points in a sphere of radius R and their corresponding weights, following Lebedev quadrature.

The number of points depends on the order, check Lebedev.jl docs.

We used `leb_order = 125` to procude N=5294 points. That was enough for all inhouse tests.
Sensors will be positioned in Far Field, therefore, `R` should be "large" - which dependent on the cloud size.

**Note**: Increase this value if your system size is already ~ R=100k₀.
"""
function get_sensors_sphereSurface(; R = 100k₀, leb_order = 125)
    x, y, z, w = lebedev_by_order(leb_order)

    xyz = zeros(3, length(x))
    for i ∈ eachindex(x)
        _inSpherical = cart2sph([x[i], y[i], z[i]])

        _inSpherical[3] = R * _inSpherical[3] # change sensor radius

        xyz[:, i] = sph2cart(_inSpherical)
    end

    return xyz, w
end


export
    laguerre

"""
    laguerre(n::Integer, α::Integer, x::Real)

Generalized Laguerre polynomials

# Examples
```jldoctest
julia> laguerre(0, 1, 5)
1

julia> laguerre(1, 3, 5)
-1
```
"""
function laguerre(n::Integer, α::Integer, x::T) where {T<:Real}
    if n == 0
        return one(T)
    elseif n == 1
        return one(T) + α - x
    else
        L_prev = one(T)
        L_k = one(T) + α - x
        for k = 1:(n-1)
            L_next = ((2k + one(T) + α - x) * L_k - (k+α) * L_prev) / (k+one(T))
            L_prev, L_k = L_k, L_next
        end
        return L_k
    end
end

"""
    laguerre(n::Integer, α::Integer)(x::Real)

Generalized Laguerre polynomials

# Examples
```jldoctest
julia> L = laguerre(1, 3);

julia> L(5)
-1
```
"""
laguerre(n::Integer, α::Integer) = x -> laguerre(n, α, x)

# function hermite(n::T, x::Real) where {T <: Integer}
#     result = 0
#     for k in 0:floor(T, n/2)
#         coeff = (-1) ^ k *
#             2 ^ (n-2k) *
#             factorial(n) / (factorial(k)*factorial(n-2k))

#         x_power = x^(n-2k)

#         result += coeff * x_power
#     end

#     return result
# end


func(x) = exp(-x)
int_func(a,b) = -exp(-b)+exp(-a)
#%%
function rect_quad(func, a, b, n)::Float64
    dx = (b-a)/n
    res = 0.
    for i in 0:(n-1)
        x = a + dx * i
        res += func(x)
    end
    return res * dx
end

function mid_quad(func::Function, a::Real, b::Real, n::Int)::Float64
    dx = (b-a)/n
    res = 0.
    for i in 0:(n-1)
        x = a + dx * (i+0.5)
        res += func(x)
    end
    return res * dx
end

function simps(func::Function, a::Real, b::Real, n::Int)::Float64
    if n % 2 !=0
         throw(ArgumentError("The number of intervals n must be even! You have choosen $n"))
     end
    dx = (b-a)/n
    res = func(a)

    w = 4
    x = 0.
    for i in 1:(n-1)
        x = a + dx*i
        res += func(x)*w
        w = 6 - w
    end
    res += func(b)
    return res * dx/3
end

function simps(func::Function, a::Real, b::Real; tol=1e-5, limit=100)
    n = 2
    iter=0
    res=0.

    while true
        res=simps(func, a, b, n)
        res1=simps(func, a, b, n*2)
        if abs(res1-res)<tol
            res=res1
            return res, n*2
        end
        n*=2
        iter+=1
        if iter>limit
            print("limit reached!")
            return NaN
        end
    end
    return NaN
end

#%%
rect_quad(func, 1., 4., 1000)
mid_quad(func, 1., 4., 1000)
simps(func, 0,10,12)
simps(func, 0, 100; tol=1e-5, limit=3)

? Dict


#!/usr/bin/env bash
    #=
    exec julia --project="$(realpath $(dirname $(dirname $0)))" --color=yes --startup-file=no -e "include(popfirst!(ARGS))" \
    "${BASH_SOURCE[0]}" "$@"
    =#

"""
```julia
hamming_distance(w₁, w₂) -> Int
```

The Hamming distance of two words is the number of changes that need to be made to each letter in the word for the words to be the same.  This does not work for words of unequal length.

Parameters:
  - `w₁`: A word.
  - `w₂`: Another word.

Returns:
  - `Int`: the number of changes needing to be made to one word for it to be identical to the other.

---

### Examples

```julia
julia> hamming_distance("ABC", "BBC") # computes the hamming distance
1
```
"""
function hamming_distance(w₁::T, w₂::T) where T <: AbstractWord
    if ! isequal(length(w₁), length(w₂))
        throw(error("Cannot compute Hamming Distance on strings of unequal length."))
    end

    distance = 0

	for (s₁, s₂) in zip(w₁, w₂)
		if s₁ ≠ s₂
			distance += 1
		end
	end

    return distance
end

function __hamming_space(relation::Function, Σⁿ::AbstractArray{T}, w::AbstractArray, e::Int) where T <: AbstractWord
	e < 0 && throw(error("e (the ball/sphere \"radius\") must be a non-negative number."))
	Σⁿ, w = deepsym(Σⁿ), ensure_symbolic(w)

	return eltype(Σⁿ)[v for v in deepsym(Σⁿ) if relation(hamming_distance(w, v), e)]
end

"""
```julia
hamming_ball(Σⁿ::AbstractArray, w::Vector, e::Int) -> Vector{Vector}
```

Get the codewords of radius ``e`` of a ball centered at word ``w``.  That is, all words whose distance from w is less than or equal to the radius.

Parameters:
  - `Σⁿ::AbstractArray`: An array of words in the code.
  - `w::Vector`: A word.
  - `e::Int`: The radius of the ball.

Returns:
  - AbstractArray: The list of words in Σⁿ whose distance from w is less than or equal to e.  Returns an array of array of symbols.

---

### Examples

```julia
julia> hamming_ball([[1, 0, 1], [0, 1, 1], [1, 0, 0]], [1, 0, 0], 2) # given a list of words, a word, and a distance e (respectively), calculate all the words in the alphabet within distance e of that word.  Converts to symbols in order to keep unique lengths
2-element Array{Any,1}:
 [Symbol("1"), Symbol("0"), Symbol("1")]
 [Symbol("1"), Symbol("0"), Symbol("0")]
```
"""
hamming_ball(Σⁿ::AbstractArray{T}, w::Vector{S}, e::Int) where {T <: AbstractWord, S} =
	__hamming_space(≤, Σⁿ, w, e)

"""
```julia
hamming_sphere(Σⁿ::AbstractArray, w::Vector, e::Int) -> Vector{Vector}
```

Get the codewords of radius e of a sohere centered at word w.  That is, all words whose distance from w is exactly equal to to the radius.

Parameters:
  - `Σⁿ::AbstractArray`: An array of words in the code.
  - `w::Vector`: A word.
  - `e::Int`: The radius of the ball.

Returns:
  - `AbstractArray`: The list of words in Σⁿ whose distance from w is exactly equal to e.  Returns an array of array of symbols.
"""
hamming_sphere(Σⁿ::AbstractArray{T}, w::Vector{S}, e::Int) where {T <: AbstractWord, S} =
	__hamming_space(isequal, Σⁿ, w, e)

"""
```julia
code_distance(C::AbstractArray) -> Int
```

Finds the distance of the code.  That is, given a code C, finds the minimum distance between any two words in the code, which are not the same. (Find the minimum hamming distance in the code for all unique letters).

Parameters:
  - `C::AbstractArray`: An array of words in the code.

Return:
  - `Int`: the distance of the code.
"""
function code_distance(C::AbstractArray{T}) where T <: AbstractWord
	isempty(C) && return nothing
	C = deepsym(C)
	min_distance = nothing

	for c in C, c′ in C
		if c ≠ c′
			ham_dist = hamming_distance(c, c′)
			if isnothing(min_distance) || ham_dist < min_distance
				min_distance = ham_dist
			end
		end
	end

	return min_distance
end

"""
```julia
code_distance!(C::AbstractArray{T}, w::T) -> Int
```

A wrapper to get the code distance after pushing a word to the code.  *This directly changes the matrix M.  Use `code_distance` for a non-mutating version of this function.*

Parameters:
  - `C::AbstractArray`: An array of words in the code.
  - `w`: A word to be appended to C.

Returns:
  - `Int`: The code distance after adding w to C.
"""
function code_distance!(C::AbstractArray{T}, w::T) where T <: AbstractWord
	push!(C, w)
	return code_distance(C)
end

"""
```julia
code_distance(C::AbstractArray{T}, w::T) -> Int
```

A wrapper to get the code distance after pushing a word to the code.

Parameters:
  - `C::AbstractArray`: An array of words in the code.
  - `w`: A word to be appended to C.

Returns:
  - `Int`: The code distance after adding w to C.

---

### Examples

```julia
julia> code_distance([[0,0,0,0,0],[1,0,1,0,1],[0,1,0,1,0],[1,1,1,1,1]]) # gets the minimum distance between two vectors in an array of vectors
2
```
"""
function code_distance(C::AbstractArray{T}, w::T) where T <: AbstractWord
	code_distance!(copy(C), w)
end

"""
```julia
t_error_detecting(C::AbstractArray{T}, t::Int) -> Bool
```

Check if a given code C can detect t many errors.

Parameters:
  - `C::AbstractArray`: An array of words in the code.
  - `t::Int`: The number of errors you want to check that the code can detect.

Returns:
  - `Bool`: Yes, C can detect t errors, or no it cannot (true of false).

---

### Examples

```julia
julia> t_error_detecting([[1, 0, 1], [0, 1, 1], [1, 0, 0]], 3)
false
```
"""
function t_error_detecting(C::AbstractArray{T}, t::Int) where T <: Union{AbstractArray{Int}, AbstractWord}
	code_distance(C) ≥ t + 1 && return true
	return false
end

"""
```julia
t_error_correcting(C::AbstractArray{T}, t::Int) -> Bool
```

Check if a given code C can correct t many errors.

Parameters:
  - `C::AbstractArray`: An array of words in the code.
  - `t::Int`: The number of errors you want to check that the code can correct.

Returns:
  - Bool: Yes, C can correct t errors, or no it cannot (true of false).
"""
function t_error_correcting(C::AbstractArray{T}, t::Int) where T <: Union{AbstractArray{Int}, AbstractWord}
	code_distance(C) ≥ 2*t + 1 && return true
	return false
end

"""
```julia
find_error_detection_max(C::AbstractArray{T}, modulo::Int) -> Int
```

Finds the greatest number t such that C is error t error detecting.

Parameters:
  - `C::AbstractArray`: An array of words in the code.
  - `moldulo::Int`: The modulus of the finite field.  The upper bound of t.

Returns:
  - `Int`: The maximum number t such that the code is t error detecting.

---

```julia
julia> find_error_detection_max([[0, 0, 0, 0], [0, 1, 1, 1], [1, 0, 1, 0], [1, 1, 0, 1]], 2)
1
```
"""
function find_error_detection_max(C::AbstractArray{T}, modulo::Int) where T <: Union{AbstractArray{Int}, AbstractWord}
	for t in (modulo - 1):-1:0
		t_error_detecting(C, t) && return t
	end
end

"""
```julia
find_error_correction_max(C::AbstractArray{T}, modulo::Int) -> Int
```

Finds the greatest number t such that C is error t error correcting.

Parameters:
  - `C::AbstractArray`: An array of words in the code.
  - `moldulo::Int`: The modulus of the finite field.  The upper bound of t.

Returns:
  - `Int`: The maximum number t such that the code is t error correcting.
"""
function find_error_correction_max(C::AbstractArray{T}, modulo::Int) where T <: Union{AbstractArray{Int}, AbstractWord}
	for t in modulo-1:-1:0
		t_error_correcting(C, t) && return t
	end
end


### A Pluto.jl notebook ###
# v0.12.21

using Markdown
using InteractiveUtils

# This Pluto notebook uses @bind for interactivity. When running this notebook outside of Pluto, the following 'mock version' of @bind gives bound variables a default value (instead of an error).
macro bind(def, element)
    quote
        local el = $(esc(element))
        global $(esc(def)) = Core.applicable(Base.get, el) ? Base.get(el) : missing
        el
    end
end

# ╔═╡ def88812-6b99-11eb-07fb-3f961d7ff6ae
begin
	using Plots
	plotly()
end

# ╔═╡ 67bd1b94-7f32-11eb-2869-5f075fca7fbc
begin
	using PlutoUI
	using StatsPlots
	using RDatasets
	using StatsBase
	using Distributions
	using LaTeXStrings
end

# ╔═╡ 253ad4f4-8253-11eb-03d8-d358d6075910
md"A mid-sem sample question:

Write a Julia snippet to sample 100 random variates denoted $\{x_1, x_2, \ldots, x_{100}\}$ from the standard normal distribution. Then compute $\dfrac{\sum_{i = 1}^{100} \sum_{j = 1}^{100} (x_i - x_j)^2}{100 \times 100}$."

# ╔═╡ 237d9290-8254-11eb-233a-4d362783d684
begin
	sample_ = rand(Normal(), 100)
	val_ = sum(sum([[(sample_[i] - sample_[j])^2 for i in 1:100] for j in 1:100]))/(100 * 100)
end

# ╔═╡ 8ce2cb16-8253-11eb-21a5-c59e5785950e
md"Run the above a few dozen times and build an intuition for what the value is for different samples in relation to $\sigma^2$ of the underlying distribution. Then deriva analytically $\mathbb{E}[(X - Y)^2]$, where $X$ and $Y$ are two independent random variables sampled from the same distribution with pdf $f$."

# ╔═╡ 22f18e82-6b7d-11eb-16eb-31758abdb1b0
md"# Notebook 6: _Measures of Two Variables_
"

# ╔═╡ 59607216-7b43-11eb-2cf7-3f74231a1839
md"
> 📕Reference material - Chapter 3: Using statistics to summarise data (Section 3.7) from Sheldon Ross.
"

# ╔═╡ 5f510e34-7f32-11eb-158a-afdded4ae4f7
iris = dataset("datasets", "iris");

# ╔═╡ 65c3f4a2-7f32-11eb-0eb4-97a2455d1ac7
diamonds = dataset("ggplot2", "diamonds");

# ╔═╡ b1716986-7e95-11eb-383d-39edf8743454
md"#### Statistics of two variables - Covariance"

# ╔═╡ 67e698c8-7e99-11eb-01c9-eb2703970ea4
md"So far we have been seeing the statistics of one variable. When there are two variables, we are interested in how they are related."

# ╔═╡ c6dc30e0-7e99-11eb-1781-7dba5912b724
md"Basic intuition is to check if increase in one of the variables also leads to an increase or decrase in another. _Do the two variables move together? Do they co-vary?_"

# ╔═╡ b79f4c20-7e99-11eb-3eba-492dc528735d
md"👉 Can you formulate a statistic that may help identify how two variables are related?"

# ╔═╡ 7b13d1a8-7e9a-11eb-39f2-5d3e9a95c811
iris_sample = select!(iris[sample(axes(iris, 1), 5; replace = false, ordered = true), :], Not(:Species))

# ╔═╡ a136b75a-7f04-11eb-1324-c9ea26353914
md"By just working with the petal length and width values, we cannot conclude if they are covarying. Notice how they are both of different scales as well."

# ╔═╡ ff01b9c2-7f02-11eb-2edd-5131a142f119
[mean(iris_sample.PetalLength) mean(iris_sample.PetalWidth)]

# ╔═╡ bbe2eec0-7f04-11eb-3f7f-594d57f134ec
md"Now that we know the mean values of the two columns are different, we can use it to compute offsets for individual values."

# ╔═╡ fc068802-7f03-11eb-36a0-f3764ac250ab
iris_offset = select(iris_sample, :PetalWidth, :PetalLength, :PetalWidth => (x -> x .- mean(x)) => :PW_offset, :PetalLength => (x -> x .- mean(x)) => :PL_offset)

# ╔═╡ eb742c80-7f04-11eb-0435-4f58142bcdc9
md"These offset values are now clearer: If two variables are to co-vary they must do so for these offset values."

# ╔═╡ 2f64101e-7f04-11eb-148b-d1c781173d4c
iris_covar = select(iris_offset, :PetalWidth, :PetalLength, :PW_offset, :PL_offset, [:PW_offset, :PL_offset] => ((x, y) -> x .* y) => :Covariance)

# ╔═╡ 98fcccfc-7f04-11eb-0b2d-6b1feaadfae4
md"All the values in the last column being positive shows that the two values actually co-vary (either both negative or both positive). A natural next step is to simply compute the average of these products."

# ╔═╡ 12e7d1a4-7f05-11eb-19d5-2b072cbafa05
mean(iris_covar.Covariance)

# ╔═╡ 2c8776c8-7f05-11eb-0226-bf1ef7542efd
md"Covariance of two sets of population values $\{X_1, X_2, \ldots, X_N\}$ and $\{Y_1, Y_2, \ldots, Y_N\}$ is given as
```math
\mathrm{COV}(X, Y) = \dfrac{\sum_{i = 1}^N (X_i - \mu_X)(Y_i - \mu_Y)}{N}
```
"

# ╔═╡ 9da0add4-7f05-11eb-1391-0f87db7d41c4
md"We can define this in terms of expectation as
```math
\mathrm{COV}(X, Y) = \mathbb{E}\left[(X - \mathbb{E}[X])(Y - \mathbb{E}[Y])\right]
```
"

# ╔═╡ a86b8dc4-7f05-11eb-3ca7-e5460fc15b96
md"Notice the parallels to the definition of $\sigma^2$ as $\mathbb{E}[(X - \mathbb{E}[X])^2]$"

# ╔═╡ 2e4af178-7f06-11eb-1247-3501f18174cc
md"👉Class exercise: Just like we rewrote $\sigma^2$ as $\mathbb{E}[X^2] - \mathbb{E}[X]^2$, can you rewrite $\mathrm{COV}(X, Y)$?"

# ╔═╡ e79a0a24-7f06-11eb-3c63-91d44fe78d49
md"Thus, we have $\mathrm{COV}(X, Y) = \mathbb{E}[XY] - \mathbb{E}[X]\mathbb{E}[Y]$"

# ╔═╡ 2be9abbe-824d-11eb-3b1d-2b1ddf16c195
md"👉 What is the covariance of two indpendent random variables $X$ and $Y$?"

# ╔═╡ 483778b4-824d-11eb-196c-2925d31dd603
md"👉 What is $\mathrm{COV}(aX, Y)$?"

# ╔═╡ b818699a-7f37-11eb-088c-a51bed3cbdc0
md"Here is a collection of some basic rules of covariance:

a) $\mathrm{COV}(X, X) = \sigma^2(X)$

b) If $X$ and $Y$ are independent then $\mathrm{COV}(X, Y) = 0$

c) $\mathrm{COV}(X, Y) = \mathrm{COV}(Y, X)$

d) $\mathrm{COV}(aX, Y) = a\mathrm{COV}(X, Y)$

e) $\mathrm{COV}(X + c, Y) = \mathrm{COV}(X, Y)$
"

# ╔═╡ 41664208-7f38-11eb-01f5-c32a33253f15
md"👉Homework exercise to prove these. All of these can be directly provded from the relation $\mathrm{COV}(X, Y) = \mathbb{E}[XY] - \mathbb{E}[X]\mathbb{E}[Y]$"

# ╔═╡ 6dfb60e6-7f38-11eb-070d-b99afff5f5a1
md"👉Class exercise: Show that $\mathrm{COV}(X + Y, Z) = \mathrm{COV}(X, Z) + \mathrm{COV}(Y, Z)$"

# ╔═╡ 2e32a1e2-7f39-11eb-0df0-812f689e1d1f
md"This rule can be generalised to the following
```math
\mathrm{COV}\left(\sum_i^m a_i X_i, \sum_j^n b_j Y_j\right) = \sum_i^m \sum_j^n a_i b_j \mathrm{COV}(X_i, Y_j)
```
"

# ╔═╡ c4d2c9d0-7f39-11eb-2ac9-373ed723a3f4
md"A neat application of the above is when we compute the variance of the sum of two random variables."

# ╔═╡ ce6afe4a-7f39-11eb-19f1-7157457700cd
md"
```math
\sigma^2(X + Y) = \mathrm{COV}(X+Y, X+Y)
```
```math
= \mathrm{COV}(X, X) + \mathrm{COV}(X, Y) + \mathrm{COV}(X, Y) + \mathrm{COV}(Y, Y)
```
```math
= \sigma^2(X) + 2\mathrm{COV}(X, Y) + \sigma^2(Y)
```
"

# ╔═╡ 175b1608-7f3a-11eb-0dfb-dd2ee73b9bac
md"Thus, we can interpret the covariance as half the value by which the variance of the sum increases from the sum of the variances for any two random variables."

# ╔═╡ fe62b640-8251-11eb-25fe-2b8aa69000ce
md"So we have seen three ways of computing covariance"

# ╔═╡ 27d06de0-7f07-11eb-23ce-4d6547ce31f6
mean(iris_covar.Covariance)

# ╔═╡ 0359bc70-7f07-11eb-2daa-d1f2a258d9e0
mean(iris_covar.PetalWidth .* iris_covar.PetalLength) - mean(iris_covar.PetalWidth) * mean(iris_covar.PetalLength)

# ╔═╡ a8c20bc6-8251-11eb-0d06-819a786e2b05
(var(iris_covar.PetalWidth .+ iris_covar.PetalLength, corrected=false) - var(iris_covar.PetalWidth, corrected=false) - var(iris_covar.PetalLength, corrected=false))/2

# ╔═╡ 9c42421a-7f0d-11eb-0172-f34b4adfa5f0
md"Notice that the covariance (like variance) has units that are product of the units of the individual variables. In the case of the iris dataset, the covariance we computed has the unit of length-squared."

# ╔═╡ 502bc23a-7f07-11eb-0b40-97e7e2a8a9ca
md"We can also directly use the `cov` function from `StatsBase`"

# ╔═╡ 2938c2ac-7f07-11eb-279c-ab02e04ac679
cov(iris_covar.PetalWidth, iris_covar.PetalLength, corrected=false)

# ╔═╡ 59828c88-7f07-11eb-1462-73e3850659fd
md"Notice we have used the `corrected = false` argument. This is because we are treating the dataset as a population. But in reality it is a sample and hence we need our correction."

# ╔═╡ a48951d0-7f07-11eb-0052-7d06cf26028c
md"Covariance of two sets of sample values $\{x_1, x_2, \ldots, x_n\}$ and $\{y_1, y_2, \ldots, y_n\}$ is given as
```math
\mathrm{COV}(x, y) = \dfrac{\sum_{i = 1}^n (x_i - \overline{x})(y_i - \overline{y})}{n - 1}
```
"

# ╔═╡ e0d54d10-7f07-11eb-214a-bd24b8accb1a
md"With this correction, the covariance is larger"

# ╔═╡ ec51f080-7f07-11eb-2559-5b4cf71a2d0e
cov(iris_covar.PetalWidth, iris_covar.PetalLength)

# ╔═╡ ef48b742-7f07-11eb-18e0-f1bcc96c86e0
md"We can then compute it for the entire dataframe instead of the small sample we were studying"

# ╔═╡ dc4d1258-7e99-11eb-25e2-ffcd05171460
cov(iris.PetalLength, iris.PetalWidth)

# ╔═╡ fa8d1044-7f07-11eb-247e-a18ecff0f4f3
md"Notice again that the large positive value of the covariance indicates that the two variable covary, This is easy to see in the following scatter plot."

# ╔═╡ 182c2ca2-7e9a-11eb-040a-21429d245b4f
scatter(iris.PetalLength, iris.PetalWidth)

# ╔═╡ 2348b81e-7f08-11eb-3070-25becdb70c5a
md"The shapes of the scatter plots reveals the covariance values."

# ╔═╡ 85e8d916-7f08-11eb-094c-17dbb38c5048
begin
	x1_ = rand(Normal(5, 2), 1000);
	y1_ = x1_ + rand(Normal(5, 1), 1000);
end

# ╔═╡ ed8c0606-7f08-11eb-13ec-d166c29a4d67
scatter(x1_, y1_, title="Covariance = " * string(cov(x1_, y1_)))

# ╔═╡ 175b39a2-7f09-11eb-02d7-c91ba9204c3c
begin
	x2_ = rand(Normal(5, 2), 1000);
	y2_ = rand(Normal(5, 2), 1000);
end

# ╔═╡ 1d4a9088-7f09-11eb-3754-07c4947ecad7
scatter(x2_, y2_, title="Covariance = " * string(cov(x2_, y2_)))

# ╔═╡ 27426ea6-7f09-11eb-3c2f-c9d5d5ed33b8
begin
	x3_ = rand(Normal(5, 2), 1000);
	y3_ = -x3_ + rand(Normal(5, 1), 1000);
end

# ╔═╡ 307bf5dc-7f09-11eb-3e23-8908558e17ca
scatter(x3_, y3_, title="Covariance = " * string(cov(x3_, y3_)))

# ╔═╡ 5a6fb146-7f09-11eb-02b3-0b2bc301eb95
md"So absolute value of covariance denotes the strength of the relationship while the sign denotes the direction (aligned is positive and oppositely aligned is negative). Two variables with 0 covariance are unrelated to each other."

# ╔═╡ 882610e4-7f09-11eb-142b-9d8ed5e88fcc
md"In real world data, we need to be careful about mixing data from different groups. For instance consider the covariance between petal length and sepal width acrsoss all flowers."

# ╔═╡ 0019040c-7e9a-11eb-3ab3-698eb0f98231
cov(iris.PetalLength, iris.SepalWidth)

# ╔═╡ 241c6342-7e9a-11eb-2053-c1984c19a023
scatter(iris.PetalLength, iris.SepalWidth)

# ╔═╡ 54ccb922-7f0a-11eb-3ba6-230b0489d928
md"The covariance value for all data together is negative. But the graph does show a positive slope."

# ╔═╡ 6273fb94-7f0a-11eb-3c1a-4dfa9d925400
md"We can gropuby the species column and compute covariance for each group separately"

# ╔═╡ b9d98800-7f09-11eb-3671-ed27dbde1c3a
combine(groupby(iris, :Species), [:PetalLength, :SepalLength] => ((x, y) -> cov(x, y))  => :Covariance)

# ╔═╡ 711d46a0-7f0a-11eb-15d7-b5b17afdeeaa
md"Each of the groups have a positive covariance (though not high) between the two variables."

# ╔═╡ 80fc7f8c-7f0a-11eb-3afd-4796a222b202
md"👉Repeat this exercise for sepal length and width variables"

# ╔═╡ 09b71b5a-7e9a-11eb-07dd-e7eabcdb50ef
cov(iris.SepalLength, iris.SepalWidth)

# ╔═╡ 2aaca4f6-7e9a-11eb-3071-fdd5483fb812
scatter(iris.SepalLength, iris.SepalWidth)

# ╔═╡ fb4be732-7f0a-11eb-2b25-b3b18d6cba8a
md"👉So far, the covariance seems like a good measure to relate two variables. Do you see a downside? Hint: When is covariance high?"

# ╔═╡ 031f2f2c-7f0c-11eb-25b7-eb7530aa62af
md"As an example consider the following two sets of covariance computations and then see the corresponding scatter plots."

# ╔═╡ 146d1aaa-7f0c-11eb-00ae-fb7ed270927b
begin
	x4_ = rand(Normal(5, 5), 1000);
	y4_ = 0.1 * x4_ + rand(Normal(5, 5), 1000);
	cov(x4_, y4_)
end

# ╔═╡ 331c74be-7f0c-11eb-1195-cf4e817f61d3
begin
	x5_ = rand(Normal(5, 0.5), 1000);
	y5_ = 1 * x5_ + rand(Normal(5, 0.5), 1000);
	cov(x5_, y5_)
end

# ╔═╡ 3cb5b896-7f0c-11eb-324d-6d745f9c3796
scatter(x4_, y4_, title="Covariance = " * string(cov(x4_, y4_)))

# ╔═╡ 4f001262-7f0c-11eb-2422-1ff23f6065a1
scatter(x5_, y5_, title="Covariance = " * string(cov(x5_, y5_)))

# ╔═╡ 6acc6cfc-7f0c-11eb-3522-174161cba17b
md"Clearly by comparing the covariance between these two sets we come to the wrong conclusion that the relationship is stronger for the first set. We thus need to **normalize** the covariance"

# ╔═╡ a8744e0a-7e95-11eb-1931-b1cdda813e0f
md"#### Statistics of two variables - Correlation"

# ╔═╡ 716cb26e-6b94-11eb-1cb0-b9c2f5608b5f
md"Correlation essentially fixes the problem we discovered - of normalizing covariance . 👉 Can you guess how?"

# ╔═╡ aff1b65e-7f0c-11eb-30de-45877373262c
md"The correlation between two sets of populations $X$ and $Y$ is given as
```math
\rho_{X, Y} = \dfrac{\mathrm{COV}(X, Y)}{\sigma_X \sigma_Y}
```
where $\sigma_X$ and $\sigma_Y$ are the standard deviations of $X$ and $Y$, respectively.
"

# ╔═╡ 3117ff70-7f0d-11eb-1c0a-95af44530f05
md"The corresponding equation for two samples $x$ and $y$ is given as
```math
\rho_{x, y} = \dfrac{\mathrm{COV}(x, y)}{s_x s_y}
```
where $s_x$ and $s_y$ are the standard deviations of $x$ and $y$, respectively.
"

# ╔═╡ a4933666-7f16-11eb-34fa-295373733b14
md"The above definition of correlation is called the _Pearson correlation coefficient_."

# ╔═╡ 5510b24e-7f0d-11eb-15a2-6714879d78f4
md"👉What is the unit of correlation?"

# ╔═╡ 93baa848-7f0e-11eb-2700-bfc3181e0f21
md"👉Class exercise: What are the minimum and maximum possible values of $\rho_{x, y}$?"

# ╔═╡ 9115a164-7f0f-11eb-3946-453ad1a450a0
md"Let us algebraically manipulate the formula of the correlation. Note that for simplicity I am dropping the limits of the summation. In all cases, they are going from $i = 1$ to $i = n$."

# ╔═╡ a731ef30-7f0e-11eb-24f0-5d102ea4a13a
md"
```math
\rho_{x, y} = \dfrac{\mathrm{COV}(x, y)}{s_x s_y} = \dfrac{\left(\sum((x_i - \overline{x})(y_i - \overline{y})\right)/(n - 1)}{\sqrt{\sum(x_i - \overline{x})^2} \sqrt{\sum(y_i - \overline{y})^2}/(n - 1)}
```
```math
\rho_{x, y}^2 = \dfrac{\left(\sum((x_i - \overline{x})(y_i - \overline{y})\right)^2}{\sum(x_i - \overline{x})^2 \sum(y_i - \overline{y})^2} = \dfrac{(\sum a_ib_i)^2}{\sum a_i^2 \sum b_i^2}
```
"

# ╔═╡ 6d579f98-7f0f-11eb-26ea-c381d36153f3
md"The Cauchy Schwarz inequality provides a bound on the RHS. It states:
```math
\left(\sum a_i b_i\right)^2 \leq \left(\sum a_i^2\right) \left(\sum b_i^2\right)
```
For a proof see [this article](https://mathworld.wolfram.com/CauchysInequality.html).
"

# ╔═╡ cdbe10ce-7f0f-11eb-303e-8f9f3527ce25
md"Applying this back, we have $\rho^2_{x, y} \leq 1$ and thus $\rho_{x, y} \in [-1, 1]$"

# ╔═╡ e33ba8e4-7f0f-11eb-36b2-039693234b96
md"👉When are the limits obtained?"

# ╔═╡ ebaeb804-7f0f-11eb-1621-216682cc1e62
md"The equality holds when $a_i = cb_i + \delta$ for all $i$ for some constant $c$. So the extreme values of correlation occur when $\rho_{x, y} = \mathrm{Sign}(c)$ if $\dfrac{(x_i - \overline{x})}{(y_i - \overline{y})} = c$ for all values in the sample"

# ╔═╡ 5adeedd8-7f11-11eb-185e-eb4b7521074b
cslider = @bind c_slider html"<input type=range min=-50 max=45 step=1>"

# ╔═╡ f1b4a228-7f10-11eb-3149-8f86ab1fd053
begin
	x6_ = rand(Normal(), 1000)
	y6_ = x6_ .* c_slider
	cor(x6_, y6_)
end

# ╔═╡ b633765a-7f10-11eb-2c80-f3cd7302ab5a
md"So, $\rho$ is unitless and has maximum and minimum values which allow us to compare across data fields or sets. We can check this with our two datasets from before."

# ╔═╡ e8bdb338-7f10-11eb-0b2d-47d7f358334f
scatter(x4_, y4_, title="Covariance = " * string(round(cov(x4_, y4_) * 1000)/1000) * ". Correlation = " * string(round(cor(x4_, y4_) * 1000)/1000), label=false)

# ╔═╡ e3cf7068-7f11-11eb-0645-1d988c17ea0f
scatter(x5_, y5_, title="Covariance = " * string(round(cov(x5_, y5_) * 1000)/1000) * ". Correlation = " * string(round(cor(x5_, y5_) * 1000)/1000), label=false)

# ╔═╡ 0e4169dc-7f12-11eb-1f09-695a9cbff347
md"The bottom case has now much higher correlation because we have scaled for the low standard deviation of the two individual variables."

# ╔═╡ 6d5a3c1e-7f12-11eb-244f-2787db2ff86b
md"We can use the `cor` function for dataframe columns also."

# ╔═╡ 4f405416-7f12-11eb-072f-b5ebf6ecdb89
cor(iris.PetalLength, iris.PetalWidth)

# ╔═╡ f678bd9a-7f12-11eb-3b82-0748786b4b75
md"For dataframes, it is usually helpful to compute pair-wise covariance and correlation between a set of quantitative variables. There is no built-in method for this, so we will use this one from [this forum](https://discourse.julialang.org/t/covariance-from-dataframe-or-timearray/48375/2)."

# ╔═╡ fd483aba-8584-11eb-1af1-fd7ba22276ba
md"👉What do you expect in the diagonals of such a matrix/"

# ╔═╡ 6c2b36ea-7f12-11eb-3b91-5f6186dd5ce3
function covmat(df)
   nc = ncol(df)
   t = zeros(nc, nc)
   for (i, c1) in enumerate(eachcol(df))
	   for (j, c2) in enumerate(eachcol(df))
		   sx, sy = skipmissings(c1, c2)
		   t[i, j] = cov(collect(sx), collect(sy))
	   end
   end
   return t
   end;

# ╔═╡ 3ae44970-7f13-11eb-3222-25121d19dfa0
covmat(select(iris, Not(:Species)))

# ╔═╡ 1387ed38-7f14-11eb-0f7b-c3deea1d8d76
md"Notice that the matrix is symmetric and the values on the diagonal are all positive"

# ╔═╡ 67b909b0-7f13-11eb-36ca-f74db0715854
propertynames(iris)

# ╔═╡ 11ef7960-8585-11eb-2dc3-7fa091595dbb
md"Now let us do the same with correlation. 👉What do you expect in the diagonals of such a matrix/"

# ╔═╡ 0376056a-7f14-11eb-0af4-45c83b91ff94
function cormat(df)
   nc = ncol(df)
   t = zeros(nc, nc)
   for (i, c1) in enumerate(eachcol(df))
	   for (j, c2) in enumerate(eachcol(df))
		   sx, sy = skipmissings(c1, c2)
		   t[i, j] = cor(collect(sx), collect(sy))
	   end
   end
   return t
   end;

# ╔═╡ 07f0fc76-7f14-11eb-14ef-ed4a723c9f39
cormat(select(iris, Not(:Species)))

# ╔═╡ 1cf0720a-7f14-11eb-28af-c73c1f94fa63
md"Notice that the values on the diagonal are now all 1. The sign of covariance and correlation is the same, but they have all been scaled up. It is easier to conclude on relatedness of variables with the correlation."

# ╔═╡ 681cbce8-7f14-11eb-30d6-17d3f7f5f116
cormat(select(diamonds, [:X, :Y, :Z, :Price]))

# ╔═╡ 5ce49422-7f14-11eb-3959-4fb5d18d1f38
md"The dimensions of the diamonds along different axes are highly correlated. Naturally the price also is correlated with the dimensions."

# ╔═╡ 76ff24b4-7f15-11eb-33c1-6118de89b12f
md"Rule of thumb: If $|\rho| > 0.5$, then we say the two samples are highly correlated. For $0.3 < |\rho| < 0.5$, we say there is moderate correlation. And for $|\rho| < 0.3$, the variables are uncorrelated or weakly correlated."

# ╔═╡ 62023c38-7f15-11eb-039c-e5020554a66f
md"👉What does a high absolute value of the correlation coefficient actually mean?"

# ╔═╡ 71394f5c-7f15-11eb-156d-27290791641f
md"If $|\rho|$ is high then we are confident of a **linear** relationship between the two samples."

# ╔═╡ 1fc41aca-7f16-11eb-331d-072d0e72cf16
md"This means that even if there is an exact _non-linear_ relationship between two variables, the correlation coefficient can be close to 0."

# ╔═╡ 35f0324a-7f16-11eb-159b-1f6056dad3f9
md"Consider the following case where there is a quadratic relationship between the two variables, but the correlation close to 0."

# ╔═╡ 577ceeb8-7f15-11eb-3700-cd396e2622b1
scatter(x5_, (x5_ .- mean(x5_)).^2, title="Correlation = " * string(cor(x5_, (x5_ .- mean(x5_)).^2)))

# ╔═╡ 59545e4e-7f16-11eb-03ae-a796f324110d
md"Thus the correlation coefficient is only a way to check a linear relatedness between variables. Often this itself is very useful in practice. For instance, does adminstering a vaccine reduce the spread of a disease? Or does adding a new modification to the engine increase range of an electric car?"

# ╔═╡ 9f311f58-7f16-11eb-2909-cb836b098441
md"But we must recognise that in most of machine learning today, we are interested in complex non-linear relationships. In such cases, the correlation coefficient may not be of much significance."

# ╔═╡ 3e702d8e-7f33-11eb-271d-5956d16a7e7d
md"👉 Can you think of a linear relationship that you would study in most ML problems?"

# ╔═╡ ed9d08b4-7f32-11eb-04b1-ef5a467dabea
md"👉Do you see a major limitation with this measure?"

# ╔═╡ 09a0093e-7f34-11eb-2210-ff1d72e9637a
md"We know that the mean (and hence offset from mean) and the standard deviation of any sample is adversely affected by outliers. So, it is likely that the Pearson correlation coefficient is also sensitive to outliers.

Notice that the argument that we are dividing the covariance by the two standard deviations is to only allow comparison across variables, not counter any potential effect of outliers."

# ╔═╡ 3fdc0494-7f34-11eb-139d-ad7fa8c098e2
begin
	x7_ = rand(Normal(5, 1), 20);
	y7_ = 1 * x7_ + rand(Normal(5, 0.3), 20);

	x8_ = append!(copy(x7_), 10)
	y8_ = append!(copy(y7_), 9)
end

# ╔═╡ 1eb842ea-7f35-11eb-3e5b-753535a8bfea
md"Click on the checkbox to add outliers"

# ╔═╡ 8608e458-7f34-11eb-3b92-290fea88ac18
scatter(x7_, y7_, title="Correlation = " * string(cor(x7_, y7_)))

# ╔═╡ bc17e118-7f34-11eb-1189-95315fab5ce6
scatter(x8_, y8_, title="Correlation = " * string(cor(x8_, y8_)))

# ╔═╡ 176b14f8-7f36-11eb-1dde-2f1ca7c9af38
md"Correlation drops significantly due to a single outlier"

# ╔═╡ 79d21678-7f36-11eb-3e14-9568a7781ebe
md"👉Can you think of a way in which correlation coefficient increases due to outliers?"

# ╔═╡ 1e02e264-7f36-11eb-12f1-81411831a1b3
begin
	x9_ = rand(Normal(5, 1), 20);
	y9_ = 0.5 * x7_ + rand(Normal(5, 2), 20);

	x10_ = append!(copy(x9_), [20])
	y10_ = append!(copy(y9_), [20])
end

# ╔═╡ a109437e-7f36-11eb-0bdf-ab2bf0b8130a
scatter(x9_, y9_, title="Correlation = " * string(cor(x9_, y9_)))

# ╔═╡ a9195216-7f36-11eb-25cf-ab7eee4ca256
scatter(x10_, y10_, title="Correlation = " * string(cor(x10_, y10_)))

# ╔═╡ ee876554-7f36-11eb-1e02-b98451ccf26f
md"👉OK so we have a problem with outliers. What would your defence be?"

# ╔═╡ 769f6de6-8585-11eb-3a9f-450df3ab75a0
md"💡Hint: What did we do in the case of mean?"

# ╔═╡ 6cd8b81a-7f17-11eb-1346-270ab26a1df4
md"#### Spearman Correlation"

# ╔═╡ e3b09fdc-7f32-11eb-3b6d-2bca422fd7f3
md"Our solution with the outlier sensitivity of mean was to use median which uses the ranking of variables rather than their values."

# ╔═╡ 14ee007e-7f37-11eb-20b4-1df87a90f48d
md"👉 Can you think of correlating two variables with ranks?"

# ╔═╡ 515ed8d2-7f3d-11eb-1f0e-f920f7799263
md"The basic idea is to compute the ranks of two set of samples and then compute the Pearson's correlation coefficient of the ranks."

# ╔═╡ 60057c6a-7f3d-11eb-0f32-595b689315a3
x12_ = rand(Normal(5, 1), 5)

# ╔═╡ afff7064-7f50-11eb-05b1-d392d177ace5
y12_ = 0.5 * x12_ + rand(Normal(5, 2), 5)

# ╔═╡ bc51fde0-7f3d-11eb-02a5-9331000ad2ff
cor(x12_, y12_)

# ╔═╡ ced6f668-7f50-11eb-2fa2-755dc4b48e74
sort(x12_)

# ╔═╡ 5aac63e2-7f50-11eb-1888-a5b1980fea98
[indexin(x12_[i], sort(x12_))[1] for i in 1:length(x12_)]

# ╔═╡ 10f3be2a-7f51-11eb-1d4f-ff3943118e1f
md"The following function replaces each value in an input array by its rank in the sorted version of the array"

# ╔═╡ d8afc632-7f50-11eb-08a5-83914cf962d0
findpos(arr) = [indexin(arr[i], sort(arr))[1] for i in 1:length(arr)]

# ╔═╡ e6cad106-7f50-11eb-1069-8520b81a05fd
x12_rank = findpos(x12_)

# ╔═╡ ed174580-7f50-11eb-29ac-e925387f1779
y12_rank = findpos(y12_)

# ╔═╡ 2222b07a-7f51-11eb-11f9-3f8c191c620e
md"Once we have the ranks for both the vectors, we can compute the Spearman correlation coefficient as the Pearson correlation coefficient on the ranks."

# ╔═╡ fcf549a2-7f50-11eb-05d3-11fbdf7f3984
cor(x12_rank, y12_rank)

# ╔═╡ 4fa95fbc-7f51-11eb-3f79-05e2c0408808
md"We can put all this together in a single function"

# ╔═╡ 3fc8a3a0-7f51-11eb-2a6c-6385c55d2d2d
cor_s(x, y) = cor(findpos(x), findpos(y))

# ╔═╡ 4ec8c7f4-7f51-11eb-133e-bbc12393f533
cor_s(x12_, y12_)

# ╔═╡ 8fbc0f54-7f52-11eb-0ab8-fb60067b43e0
md"Let us now revisit the examples where outliers sensitively affected the Pearson correlation coefficient"

# ╔═╡ a97b5ff8-7f52-11eb-220e-55f4aceb21de
md"Case 1: ρ-Pearson sharply reduced by a single outlier."

# ╔═╡ b2052776-7f52-11eb-3790-ff5c917d9da7
md"Case 2: ρ-Pearson sharply increased by an outlier"

# ╔═╡ ddad5ac6-7f52-11eb-148b-91e5b7a5df8f
md"In both cases the Spearman correlation coefficient remains more robust in the face of the outliers. This is clear why - the outliers have only limited effect on the rank of the values which are then used to compute correlation."

# ╔═╡ c17e66ac-8585-11eb-2cc8-3fd3501f33ce
md"👉Do you see one other advantage of Spearman correlation coefficient over the Pearson correlation coefficient?"

# ╔═╡ 0959a256-7f53-11eb-2afe-c90157d032f3
md"Since the Spearman correlation depends only on the rank ordering it can also detect correlations with non-linear _monotonic_ functions."

# ╔═╡ 73f12006-7f53-11eb-23e9-47e725e79336
begin
	x16_ = rand(Uniform(-10, 10), 100)
	y16_ = sinh.(x16_)
end

# ╔═╡ 5684ace4-7f54-11eb-197a-bd770705eced
md"👉Where is this particularly important?"

# ╔═╡ 0868668c-7f53-11eb-0025-eb7e12f385dc
hint(text, title) = Markdown.MD(Markdown.Admonition("hint", title, [text]))

# ╔═╡ c6efb902-8255-11eb-16f0-6ffc51cf9224
hint(md"
```julia
begin
	sample_ = rand(Normal(), 100)
	val_ = sum(sum([[(sample_[i] - sample_[j])^2 for i in 1:100] for j in 1:100]))/(100 * 100)
end
```", "Solution")

# ╔═╡ 1dc8ee70-8255-11eb-3d0e-53dda549c76b
hint(md"$\mathbb{E}[(X - Y)^2] = \mathbb{E}[X^2] + \mathbb{E}[Y^2] - 2\mathbb{E}[XY]$ But we know that $X$ and $Y$ are independent and so $\mathbb{E}[XY] = \mathbb{E}[X]\mathbb{E}[Y] = \mu^2$. We also know that $\mathbb{E}[X^2] = {E}[Y^2] = \mu^2 + \sigma^2$. Hence, the above expectation is equal to $2\sigma^2$. Hence, it makes sense to get values around 2 for the previous code.", "Solution")

# ╔═╡ 5acfe6ae-7f06-11eb-182b-9b83107e0c4c
hint(md"
```math
\mathbb{E}[(X - \mathbb{E}[X])(Y - \mathbb{E}[Y])] = \mathbb{E}[XY - X\mathbb{E}[Y] - Y\mathbb{E}[X] + \mathbb{E}[X]\mathbb{E}[Y]]
```
```math
= \mathbb{E}[XY] - \mathbb{E}[X]\mathbb{E}[Y] - \mathbb{E}[Y]\mathbb{E}[X] + \mathbb{E}[X]\mathbb{E}[Y] =  \mathbb{E}[XY] - \mathbb{E}[X]\mathbb{E}[Y]
```", "Rewriting covariance")

# ╔═╡ ac68800c-7f38-11eb-0c1b-79e8d2ab8ecf
hint(md"
```math
\mathrm{COV}(X + Y, Z) = \mathbb{E}[(X + Y)Z] - \mathbb{E}[X+Y]\mathbb{E}[Z]
```
```math
= \mathbb{E}[XZ] + \mathbb{E}[YZ] - \mathbb{E}[X]\mathbb{E}[Z] - \mathbb{E}[Y]\mathbb{E}[Z]
```
```math
= (\mathbb{E}[XZ] - \mathbb{E}[X]\mathbb{E}[Z]) + (\mathbb{E}[YZ]  - \mathbb{E}[Y]\mathbb{E}[Z])
```
```math
= \mathrm{COV}(X, Z) + \mathrm{COV}(Y, Z)
```", "Covariance sum")

# ╔═╡ baa84258-7f0b-11eb-03d4-c5e0d9547508
hint(md"Covariance is high when the signs of the offsets of the two variable match often (good part), but also when the two variables have large variances individually (bad part)", "Issue with covariance")

# ╔═╡ c6bd6268-7f0d-11eb-2094-0fdc2a8fbd88
hint(md"Correlation is unit-less as we have scaled the product of offsets from mean by the product of two standard deviations.", "Unit of covariance")

# ╔═╡ 48065f1c-7f33-11eb-1888-13ba63eea532
hint(md"Usually whenever we do regression, we have the actual value and the estimate. And we do expect an identity relation between them. In such cases we use the coefficient of determination $R^2$ as a goodness of fit. But in some cases we are fine with a high correlation between the actual and estimated value.", "Use of correlation in ML")

# ╔═╡ 5ff7afb0-7f54-11eb-0f8c-bb7af9a0de90
hint(md"Let us say we are building a ML estimator for efficacy of different manufacturing parameters for yield. Even if this function is non-linear but monotonic in the design parameters, then we can use the Spearman correlation coefficient to set the parameters for increasing the yield..", "Spearman correlation - usage")

# ╔═╡ 14eca90a-7f52-11eb-1257-c3eb50212780
disp(number) = string(round(number*1000)/1000)

# ╔═╡ 58917ca4-7f51-11eb-2399-53cc4269ba1b
scatter(x7_, y7_, title="ρ-Pearson = " * disp(cor(x7_, y7_)) * " ρ-Spearman = " * disp(cor_s(x7_, y7_)))

# ╔═╡ 37d45bea-7f52-11eb-2302-2da751a91c33
scatter(x8_, y8_, title="ρ-Pearson = " * disp(cor(x8_, y8_)) * " ρ-Spearman = " * disp(cor_s(x8_, y8_)))

# ╔═╡ c1ab36ca-7f52-11eb-2965-8914dacd089b
scatter(x9_, y9_, title="ρ-Pearson = " * disp(cor(x9_, y9_)) * " ρ-Spearman = " * disp(cor_s(x9_, y9_)))

# ╔═╡ cc35a918-7f52-11eb-1bc5-db77f47abf9f
scatter(x10_, y10_, title="ρ-Pearson = " * disp(cor(x10_, y10_)) * " ρ-Spearman = " * disp(cor_s(x10_, y10_)))

# ╔═╡ 8494b7a6-7f53-11eb-0554-e3f7afb68fc5
scatter(x16_, y16_, title="ρ-Pearson = " * disp(cor(x16_, y16_)) * " ρ-Spearman = " * disp(cor_s(x16_, y16_)), legend=false)

# ╔═╡ Cell order:
# ╟─253ad4f4-8253-11eb-03d8-d358d6075910
# ╟─c6efb902-8255-11eb-16f0-6ffc51cf9224
# ╠═237d9290-8254-11eb-233a-4d362783d684
# ╟─8ce2cb16-8253-11eb-21a5-c59e5785950e
# ╟─1dc8ee70-8255-11eb-3d0e-53dda549c76b
# ╟─22f18e82-6b7d-11eb-16eb-31758abdb1b0
# ╟─59607216-7b43-11eb-2cf7-3f74231a1839
# ╠═def88812-6b99-11eb-07fb-3f961d7ff6ae
# ╠═67bd1b94-7f32-11eb-2869-5f075fca7fbc
# ╠═5f510e34-7f32-11eb-158a-afdded4ae4f7
# ╠═65c3f4a2-7f32-11eb-0eb4-97a2455d1ac7
# ╟─b1716986-7e95-11eb-383d-39edf8743454
# ╟─67e698c8-7e99-11eb-01c9-eb2703970ea4
# ╟─c6dc30e0-7e99-11eb-1781-7dba5912b724
# ╟─b79f4c20-7e99-11eb-3eba-492dc528735d
# ╠═7b13d1a8-7e9a-11eb-39f2-5d3e9a95c811
# ╟─a136b75a-7f04-11eb-1324-c9ea26353914
# ╠═ff01b9c2-7f02-11eb-2edd-5131a142f119
# ╟─bbe2eec0-7f04-11eb-3f7f-594d57f134ec
# ╠═fc068802-7f03-11eb-36a0-f3764ac250ab
# ╟─eb742c80-7f04-11eb-0435-4f58142bcdc9
# ╠═2f64101e-7f04-11eb-148b-d1c781173d4c
# ╟─98fcccfc-7f04-11eb-0b2d-6b1feaadfae4
# ╠═12e7d1a4-7f05-11eb-19d5-2b072cbafa05
# ╟─2c8776c8-7f05-11eb-0226-bf1ef7542efd
# ╟─9da0add4-7f05-11eb-1391-0f87db7d41c4
# ╟─a86b8dc4-7f05-11eb-3ca7-e5460fc15b96
# ╟─2e4af178-7f06-11eb-1247-3501f18174cc
# ╟─5acfe6ae-7f06-11eb-182b-9b83107e0c4c
# ╟─e79a0a24-7f06-11eb-3c63-91d44fe78d49
# ╟─2be9abbe-824d-11eb-3b1d-2b1ddf16c195
# ╟─483778b4-824d-11eb-196c-2925d31dd603
# ╟─b818699a-7f37-11eb-088c-a51bed3cbdc0
# ╟─41664208-7f38-11eb-01f5-c32a33253f15
# ╟─6dfb60e6-7f38-11eb-070d-b99afff5f5a1
# ╟─ac68800c-7f38-11eb-0c1b-79e8d2ab8ecf
# ╟─2e32a1e2-7f39-11eb-0df0-812f689e1d1f
# ╟─c4d2c9d0-7f39-11eb-2ac9-373ed723a3f4
# ╟─ce6afe4a-7f39-11eb-19f1-7157457700cd
# ╟─175b1608-7f3a-11eb-0dfb-dd2ee73b9bac
# ╟─fe62b640-8251-11eb-25fe-2b8aa69000ce
# ╠═27d06de0-7f07-11eb-23ce-4d6547ce31f6
# ╠═0359bc70-7f07-11eb-2daa-d1f2a258d9e0
# ╠═a8c20bc6-8251-11eb-0d06-819a786e2b05
# ╟─9c42421a-7f0d-11eb-0172-f34b4adfa5f0
# ╟─502bc23a-7f07-11eb-0b40-97e7e2a8a9ca
# ╠═2938c2ac-7f07-11eb-279c-ab02e04ac679
# ╟─59828c88-7f07-11eb-1462-73e3850659fd
# ╟─a48951d0-7f07-11eb-0052-7d06cf26028c
# ╟─e0d54d10-7f07-11eb-214a-bd24b8accb1a
# ╠═ec51f080-7f07-11eb-2559-5b4cf71a2d0e
# ╟─ef48b742-7f07-11eb-18e0-f1bcc96c86e0
# ╠═dc4d1258-7e99-11eb-25e2-ffcd05171460
# ╟─fa8d1044-7f07-11eb-247e-a18ecff0f4f3
# ╠═182c2ca2-7e9a-11eb-040a-21429d245b4f
# ╟─2348b81e-7f08-11eb-3070-25becdb70c5a
# ╠═85e8d916-7f08-11eb-094c-17dbb38c5048
# ╠═ed8c0606-7f08-11eb-13ec-d166c29a4d67
# ╠═175b39a2-7f09-11eb-02d7-c91ba9204c3c
# ╠═1d4a9088-7f09-11eb-3754-07c4947ecad7
# ╠═27426ea6-7f09-11eb-3c2f-c9d5d5ed33b8
# ╠═307bf5dc-7f09-11eb-3e23-8908558e17ca
# ╟─5a6fb146-7f09-11eb-02b3-0b2bc301eb95
# ╟─882610e4-7f09-11eb-142b-9d8ed5e88fcc
# ╠═0019040c-7e9a-11eb-3ab3-698eb0f98231
# ╠═241c6342-7e9a-11eb-2053-c1984c19a023
# ╟─54ccb922-7f0a-11eb-3ba6-230b0489d928
# ╟─6273fb94-7f0a-11eb-3c1a-4dfa9d925400
# ╠═b9d98800-7f09-11eb-3671-ed27dbde1c3a
# ╟─711d46a0-7f0a-11eb-15d7-b5b17afdeeaa
# ╟─80fc7f8c-7f0a-11eb-3afd-4796a222b202
# ╠═09b71b5a-7e9a-11eb-07dd-e7eabcdb50ef
# ╠═2aaca4f6-7e9a-11eb-3071-fdd5483fb812
# ╟─fb4be732-7f0a-11eb-2b25-b3b18d6cba8a
# ╟─baa84258-7f0b-11eb-03d4-c5e0d9547508
# ╟─031f2f2c-7f0c-11eb-25b7-eb7530aa62af
# ╠═146d1aaa-7f0c-11eb-00ae-fb7ed270927b
# ╠═331c74be-7f0c-11eb-1195-cf4e817f61d3
# ╠═3cb5b896-7f0c-11eb-324d-6d745f9c3796
# ╠═4f001262-7f0c-11eb-2422-1ff23f6065a1
# ╟─6acc6cfc-7f0c-11eb-3522-174161cba17b
# ╟─a8744e0a-7e95-11eb-1931-b1cdda813e0f
# ╟─716cb26e-6b94-11eb-1cb0-b9c2f5608b5f
# ╟─aff1b65e-7f0c-11eb-30de-45877373262c
# ╟─3117ff70-7f0d-11eb-1c0a-95af44530f05
# ╟─a4933666-7f16-11eb-34fa-295373733b14
# ╟─5510b24e-7f0d-11eb-15a2-6714879d78f4
# ╟─c6bd6268-7f0d-11eb-2094-0fdc2a8fbd88
# ╟─93baa848-7f0e-11eb-2700-bfc3181e0f21
# ╟─9115a164-7f0f-11eb-3946-453ad1a450a0
# ╟─a731ef30-7f0e-11eb-24f0-5d102ea4a13a
# ╟─6d579f98-7f0f-11eb-26ea-c381d36153f3
# ╟─cdbe10ce-7f0f-11eb-303e-8f9f3527ce25
# ╟─e33ba8e4-7f0f-11eb-36b2-039693234b96
# ╟─ebaeb804-7f0f-11eb-1621-216682cc1e62
# ╠═5adeedd8-7f11-11eb-185e-eb4b7521074b
# ╠═f1b4a228-7f10-11eb-3149-8f86ab1fd053
# ╟─b633765a-7f10-11eb-2c80-f3cd7302ab5a
# ╟─e8bdb338-7f10-11eb-0b2d-47d7f358334f
# ╟─e3cf7068-7f11-11eb-0645-1d988c17ea0f
# ╟─0e4169dc-7f12-11eb-1f09-695a9cbff347
# ╟─6d5a3c1e-7f12-11eb-244f-2787db2ff86b
# ╠═4f405416-7f12-11eb-072f-b5ebf6ecdb89
# ╟─f678bd9a-7f12-11eb-3b82-0748786b4b75
# ╟─fd483aba-8584-11eb-1af1-fd7ba22276ba
# ╠═6c2b36ea-7f12-11eb-3b91-5f6186dd5ce3
# ╠═3ae44970-7f13-11eb-3222-25121d19dfa0
# ╟─1387ed38-7f14-11eb-0f7b-c3deea1d8d76
# ╠═67b909b0-7f13-11eb-36ca-f74db0715854
# ╟─11ef7960-8585-11eb-2dc3-7fa091595dbb
# ╠═0376056a-7f14-11eb-0af4-45c83b91ff94
# ╠═07f0fc76-7f14-11eb-14ef-ed4a723c9f39
# ╟─1cf0720a-7f14-11eb-28af-c73c1f94fa63
# ╠═681cbce8-7f14-11eb-30d6-17d3f7f5f116
# ╟─5ce49422-7f14-11eb-3959-4fb5d18d1f38
# ╟─76ff24b4-7f15-11eb-33c1-6118de89b12f
# ╟─62023c38-7f15-11eb-039c-e5020554a66f
# ╟─71394f5c-7f15-11eb-156d-27290791641f
# ╟─1fc41aca-7f16-11eb-331d-072d0e72cf16
# ╟─35f0324a-7f16-11eb-159b-1f6056dad3f9
# ╠═577ceeb8-7f15-11eb-3700-cd396e2622b1
# ╟─59545e4e-7f16-11eb-03ae-a796f324110d
# ╟─9f311f58-7f16-11eb-2909-cb836b098441
# ╟─3e702d8e-7f33-11eb-271d-5956d16a7e7d
# ╟─48065f1c-7f33-11eb-1888-13ba63eea532
# ╟─ed9d08b4-7f32-11eb-04b1-ef5a467dabea
# ╟─09a0093e-7f34-11eb-2210-ff1d72e9637a
# ╠═3fdc0494-7f34-11eb-139d-ad7fa8c098e2
# ╟─1eb842ea-7f35-11eb-3e5b-753535a8bfea
# ╠═8608e458-7f34-11eb-3b92-290fea88ac18
# ╠═bc17e118-7f34-11eb-1189-95315fab5ce6
# ╟─176b14f8-7f36-11eb-1dde-2f1ca7c9af38
# ╟─79d21678-7f36-11eb-3e14-9568a7781ebe
# ╠═1e02e264-7f36-11eb-12f1-81411831a1b3
# ╠═a109437e-7f36-11eb-0bdf-ab2bf0b8130a
# ╠═a9195216-7f36-11eb-25cf-ab7eee4ca256
# ╟─ee876554-7f36-11eb-1e02-b98451ccf26f
# ╟─769f6de6-8585-11eb-3a9f-450df3ab75a0
# ╟─6cd8b81a-7f17-11eb-1346-270ab26a1df4
# ╟─e3b09fdc-7f32-11eb-3b6d-2bca422fd7f3
# ╟─14ee007e-7f37-11eb-20b4-1df87a90f48d
# ╟─515ed8d2-7f3d-11eb-1f0e-f920f7799263
# ╠═60057c6a-7f3d-11eb-0f32-595b689315a3
# ╠═afff7064-7f50-11eb-05b1-d392d177ace5
# ╠═bc51fde0-7f3d-11eb-02a5-9331000ad2ff
# ╠═ced6f668-7f50-11eb-2fa2-755dc4b48e74
# ╠═5aac63e2-7f50-11eb-1888-a5b1980fea98
# ╟─10f3be2a-7f51-11eb-1d4f-ff3943118e1f
# ╠═d8afc632-7f50-11eb-08a5-83914cf962d0
# ╠═e6cad106-7f50-11eb-1069-8520b81a05fd
# ╠═ed174580-7f50-11eb-29ac-e925387f1779
# ╟─2222b07a-7f51-11eb-11f9-3f8c191c620e
# ╠═fcf549a2-7f50-11eb-05d3-11fbdf7f3984
# ╟─4fa95fbc-7f51-11eb-3f79-05e2c0408808
# ╠═3fc8a3a0-7f51-11eb-2a6c-6385c55d2d2d
# ╠═4ec8c7f4-7f51-11eb-133e-bbc12393f533
# ╟─8fbc0f54-7f52-11eb-0ab8-fb60067b43e0
# ╟─a97b5ff8-7f52-11eb-220e-55f4aceb21de
# ╟─58917ca4-7f51-11eb-2399-53cc4269ba1b
# ╟─37d45bea-7f52-11eb-2302-2da751a91c33
# ╟─b2052776-7f52-11eb-3790-ff5c917d9da7
# ╟─c1ab36ca-7f52-11eb-2965-8914dacd089b
# ╟─cc35a918-7f52-11eb-1bc5-db77f47abf9f
# ╟─ddad5ac6-7f52-11eb-148b-91e5b7a5df8f
# ╟─c17e66ac-8585-11eb-2cc8-3fd3501f33ce
# ╟─0959a256-7f53-11eb-2afe-c90157d032f3
# ╠═73f12006-7f53-11eb-23e9-47e725e79336
# ╠═8494b7a6-7f53-11eb-0554-e3f7afb68fc5
# ╟─5684ace4-7f54-11eb-197a-bd770705eced
# ╟─5ff7afb0-7f54-11eb-0f8c-bb7af9a0de90
# ╠═0868668c-7f53-11eb-0025-eb7e12f385dc
# ╠═14eca90a-7f52-11eb-1257-c3eb50212780


Eye4f0() = Mat4f0(Matrix(1.0f0I, 4, 4))
#Came from GLAbstraction/GLMatrixmath.jl
function scalemat(s::Vec{3,T}) where {T}
    T0, T1 = zero(T), one(T)
    return Mat{4}(s[1], T0, T0, T0, T0, s[2], T0, T0, T0, T0, s[3], T0, T0, T0, T0, T1)
end

function scalemat(s::T) where {T}
    T0, T1 = zero(T), one(T)
    return Mat{4}(s, T0, T0, T0, T0, s, T0, T0, T0, T0, s, T0, T0, T0, T0, T1)
end

scalemat(s::Point{3,T}) where {T} = scalemat(convert(Vec{3,T}, s))

translmat_x(x::T) where {T} = translmat(Vec{3,T}(x, 0, 0))
translmat_y(y::T) where {T} = translmat(Vec{3,T}(0, y, 0))
translmat_z(z::T) where {T} = translmat(Vec{3,T}(0, 0, z))

function translmat(t::Vec{3,T}) where {T}
    T0, T1 = zero(T), one(T)
    return Mat{4}(T1, T0, T0, T0, T0, T1, T0, T0, T0, T0, T1, T0, t[1], t[2], t[3], T1)
end
translmat(t::Point{3,T}) where {T} = translmat(convert(Vec{3,T}, t))

function rotate(a::Number, axis::Vec{3,T}) where {T}
    axis = normalize(axis)
    u, v, w = axis
    return Mat4{T}(u^2 + (1 - u^2) * cos(a), u * v * (1 - cos(a)) + w * sin(a),
                   u * w * (1 - cos(a)) - v * sin(a), 0.0,
                   u * v * (1 - cos(a)) - w * sin(a), v^2 + (1 - v^2) * cos(a),
                   v * w * (1 - cos(a)) + u * sin(a), 0.0,
                   u * w * (1 - cos(a)) + v * sin(a), v * w * (1 - cos(a)) - u * sin(a),
                   w^2 + (1 - w^2) * cos(a), 0.0, 0.0, 0.0, 0.0, 1.0)
end

function rotate(::Type{T}, angle::Number, axis::Vec{3}) where {T}
    return rotate(T(angle), convert(Vec{3,T}, axis))
end

function rotate(v1::T, angle::Number, axis) where {T<:StaticArray{Tuple{3}}}
    return T((rotate(angle, axis)*SVector{4}(v1..., 1.0))[1:3]...)
end
function rotate(v1::T, angle::Number, origin, axis) where {T<:StaticArray{Tuple{3}}}
    return T((translmat(origin)*rotate(angle, axis)*translmat(-origin)*SVector{4}(v1...,
                                                                                1.0))[1:3]...)
end

function rotate(v1::Vec3{T}, v2::Vec3{T}) where {T}
    vr = v2 - v1
    l = norm(vr)
    angle = acos(vr[3] / l)
    axis = normalize(cross(Vec3(0.0000000001f0, 0.0f0, 1.0f0), vr))
    if length(axis) > 0.0001
        return rotate(angle, axis)
    end
    return Mat4(Eye4f0(T, 4))
end

function rotmat_x(angle::T) where {T}
    T0, T1 = zero(T), one(T)
    return Mat{4}(T1, T0, T0, T0, T0, cos(angle), sin(angle), T0, T0, -sin(angle),
                  cos(angle), T0, T0, T0, T0, T1)
end
function rotmat_y(angle::T) where {T}
    T0, T1 = zero(T), one(T)
    return Mat{4}(cos(angle), T0, -sin(angle), T0, T0, T1, T0, T0, sin(angle), T0,
                  cos(angle), T0, T0, T0, T0, T1)
end
function rotmat_z(angle::T) where {T}
    T0, T1 = zero(T), one(T)
    return Mat{4}(cos(angle), sin(angle), T0, T0, -sin(angle), cos(angle), T0, T0, T0, T0,
                  T1, T0, T0, T0, T0, T1)
end
"""
    Create view frustum

    Parameters
    ----------
        left : float
         Left coordinate of the field of view.
        right : float
         Left coordinate of the field of view.
        bottom : float
         Bottom coordinate of the field of view.
        top : float
         Top coordinate of the field of view.
        znear : float
         Near coordinate of the field of view.
        zfar : float
         Far coordinate of the field of view.

    Returns
    -------
        M : array
         View frustum matrix (4x4).
"""
function frustum(left::T, right::T, bottom::T, top::T, znear::T, zfar::T) where {T}
    (right == left || bottom == top || znear == zfar) && return Mat4{T}(I)
    T0, T1, T2 = zero(T), one(T), T(2)
    return Mat4{T}(T2 * znear / (right - left), T0, T0, T0, T0, T2 * znear / (top - bottom),
                   T0, T0, (right + left) / (right - left), (top + bottom) / (top - bottom),
                   -(zfar + znear) / (zfar - znear), -T1, T0, T0,
                   -(T2 * znear * zfar) / (zfar - znear), T0)
end

"""
`proj = projmatpersp([T], fovy, aspect, znear, zfar)` defines
a projection matrix with a given angular field-of-view `fovy` along
the y-axis (measured in degrees), the specified `aspect` ratio, and
near and far clipping planes `znear`, `zfar`. Optionally specify the
element type `T` of the matrix.
"""
function projmatpersp(fovy::T, aspect::T, znear::T, zfar::T) where {T}
    (znear == zfar) && error("znear ($znear) must be different from tfar ($zfar)")
    h = T(tan(fovy / 180.0 * pi) * znear)
    w = T(h * aspect)
    return frustum(-w, w, -h, h, znear, zfar)
end

function projmatpersp(w::Integer, h::Integer, fovy::T, znear::T, zfar::T) where {T}
    return projmatpersp(fovy, T(w / h), znear, zfar)
end

function projmatpersp(::Type{T}, fovy::Number, aspect::Number, znear::Number,
                      zfar::Number) where {T}
    return projmatpersp(T(fovy), T(aspect), T(znear), T(zfar))
end
"""
`proj = projmatpersp([T], rect, fov, near, far)` defines the
projection ratio in terms of the rectangular view size `rect` rather
than the aspect ratio.
"""
function projmatpersp(wh::Area, fov::T, near::T, far::T) where {T}
    return projmatpersp(fov, T(wh.w / wh.h), near, far)
end
function projmatpersp(::Type{T}, wh::Area, fov::Number, near::Number, far::Number) where {T}
    return projmatpersp(T(fov), T(wh.w / wh.h), T(near), T(far))
end

"""
`view = lookatmat(eyeposition, lookatmat, up)` creates a view matrix with
the eye located at `eyeposition` and looking at position `lookatmat`,
with the top of the window corresponding to the direction `up`. Only
the component of `up` that is perpendicular to the vector pointing
from `eyeposition` to `lookatmat` will be used.  All inputs must be
supplied as 3-vectors.
"""
function lookatmat(eyePos::Union{Vec{3,T},Point{3,T}}, lookAt::Union{Vec{3,T},Point{3,T}},
                   up::Union{Vec{3,T},Point{3,T}}) where {T}
    zaxis  = normalize(eyePos - lookAt)
    xaxis  = normalize(cross(up, zaxis))
    yaxis  = normalize(cross(zaxis, xaxis))
    T0, T1 = zero(T), one(T)
    return Mat{4}(xaxis[1], yaxis[1], zaxis[1], T0, xaxis[2], yaxis[2], zaxis[2], T0,
                  xaxis[3], yaxis[3], zaxis[3], T0, T0, T0, T0, T1) * translmat(-eyePos)
end
function lookatmat(eyePos, lookAt::Union{Vec{3,T},Point{3,T}},
                   up::Union{Vec{3,T},Point{3,T}}) where {T}
    return lookatmat(Vec{3,T}(eyePos), lookAt, up)
end
function lookatmat(::Type{T}, eyePos, lookAt::Vec{3}, up::Vec{3}) where {T}
    return lookatmat(Vec{3,T}(eyePos), Vec{3,T}(lookAt), Vec{3,T}(up))
end

"""
    projmatortho(left::T, right::T, bottom::T, top::T) where T

2D orthographic projection.
"""
function projmatortho(left::T, right::T, bottom::T, top::T) where {T}
    out = zeros(T, 4, 4)
    out[1, 1] = 2 / (right - left)
    out[2, 2] = 2 / (top - bottom)
    out[3, 3] = -1
    out[4, 1] = -(right + left) / (right - left)
    out[4, 2] = -(top + bottom) / (top - bottom)
    out[4, 4] = 1
    return Mat4{T}(out')
end

function projmatortho(::Type{T}, left::Number, right::Number, bottom::Number,
                      top::Number) where {T}
    return projmatortho(T(left), T(right), T(bottom), T(top))
end

function projmatortho(wh::Area, near::T, far::T) where {T}
    return projmatortho(zero(T), T(wh.w), zero(T), T(wh.h), near, far)
end
function projmatortho(::Type{T}, wh::Area, near::Number, far::Number) where {T}
    return projmatortho(wh, T(near), T(far))
end
function projmatortho(w::Integer, h::Integer, near::T, far::T) where {T}
    return projmatortho(zero(T), T(w), zero(T), T(h), near, far)
end

"""
    projmatortho(left::T, right::T, bottom::T, top::T, znear::T, zfar::T) where T

3D orthographic projection.
"""
function projmatortho(left::T, right::T, bottom::T, top::T, znear::T, zfar::T) where {T}
    if right == left || bottom == top || znear == zfar
        return Mat4{T}(I)
    else
        out = zeros(T, 4, 4)
        out[1, 1] = 2 / (right - left)
        out[2, 2] = 2 / (top - bottom)
        out[3, 3] = -2 / (zfar - znear)
        out[4, 1] = -(right + left) / (right - left)
        out[4, 2] = -(top + bottom) / (top - bottom)
        out[4, 3] = -(zfar + znear) / (zfar - znear)
        out[4, 4] = 1
        return Mat4{T}(out')
    end
end

function projmatortho(::Type{T}, left::Number, right::Number, bottom::Number, top::Number,
                      znear::Number, zfar::Number) where {T}
    return projmatortho(T(left), T(right), T(bottom), T(top), T(znear), T(zfar))
end

function Base.:(*)(q::Quaternions.Quaternion{T}, v::Vec{3,T}) where {T}
    t = T(2) * cross(Vec(q.v1, q.v2, q.v3), v)
    return v + q.s * t + cross(Vec(q.v1, q.v2, q.v3), t)
end
function Quaternions.qrotation(axis::Vec{3,T}, theta) where {T<:Real}
    u = normalize(axis)
    s = sin(theta / 2)
    return Quaternions.Quaternion{T}(cos(theta / 2), s * u[1], s * u[2], s * u[3], true)
end

mutable struct Pivot{T}
    origin      :: Vec{3,T}
    xaxis       :: Vec{3,T}
    yaxis       :: Vec{3,T}
    zaxis       :: Vec{3,T}
    rotation    :: Quaternions.Quaternion
    translation :: Vec{3,T}
    scale       :: Vec{3,T}
end

GeometryBasics.origin(p::Pivot) = p.origin

rotmat4(q::Quaternions.Quaternion{T}) where {T} = Mat4{T}(q)

function (::Type{M})(q::Quaternions.Quaternion) where {M<:Mat4}
    T = eltype(M)
    sx, sy, sz = 2q.s * q.v1, 2q.s * q.v2, 2q.s * q.v3
    xx, xy, xz = 2q.v1^2, 2q.v1 * q.v2, 2q.v1 * q.v3
    yy, yz, zz = 2q.v2^2, 2q.v2 * q.v3, 2q.v3^2
    T0, T1 = zero(T), one(T)
    return Mat{4}(T1 - (yy + zz), xy + sz, xz - sy, T0, xy - sz, T1 - (xx + zz), yz + sx,
                  T0, xz + sy, yz - sx, T1 - (xx + yy), T0, T0, T0, T0, T1)
end

function (::Type{M})(q::Quaternions.Quaternion) where {M<:Mat3}
    T = eltype(M)
    sx, sy, sz = 2q.s * q.v1, 2q.s * q.v2, 2q.s * q.v3
    xx, xy, xz = 2q.v1^2, 2q.v1 * q.v2, 2q.v1 * q.v3
    yy, yz, zz = 2q.v2^2, 2q.v2 * q.v3, 2q.v3^2
    T0, T1 = zero(T), one(T)
    return Mat{3}(T1 - (yy + zz), xy + sz, xz - sy, xy - sz, T1 - (xx + zz), yz + sx,
                  xz + sy, yz - sx, T1 - (xx + yy))
end
function transfmat(p::Pivot)
    return translmat(p.origin) * #go to origin
           rotmat4(p.rotation) * #apply rotation
           translmat(-p.origin) * # go back to origin
           translmat(p.translation) #apply translation
end

function transfmat(translation, scale)
    T = eltype(translation)
    T0, T1 = zero(T), one(T)
    return Mat{4}(scale[1], T0, T0, T0, T0, scale[2], T0, T0, T0, T0, scale[3], T0,
                  translation[1], translation[2], translation[3], T1)
end

function transfmat(translation, scale, rotation::Quaternions.Quaternion)
    T = eltype(translation)
    trans_scale = transfmat(translation, scale)
    rotation = Mat4f0(rotation)
    return trans_scale * rotation
end
function transfmat(translation, scale, rotation::Vec{3,T}, up = Vec{3,T}(0, 0, 1)) where {T}
    q = rotation(rotation, up)
    return transfmat(translation, scale, q)
end

#Calculate rotation between two vectors
function rotation(u::Vec{3,T}, v::Vec{3,T}) where {T}
    # It is important that the inputs are of equal length when
    # calculating the half-way vector.
    u, v = normalize(u), normalize(v)
    # Unfortunately, we have to check for when u == -v, as u + v
    # in this case will be (0, 0, 0), which cannot be normalized.
    if (u == -v)
        # 180 degree rotation around any orthogonal vector
        other = (abs(dot(u, Vec{3,T}(1, 0, 0))) < 1.0) ? Vec{3,T}(1, 0, 0) :
                Vec{3,T}(0, 1, 0)
        return Quaternions.qrotation(normalize(cross(u, other)), T(180))
    end
    half = normalize(u + v)
    return Quaternions.Quaternion(dot(u, half), cross(u, half)...)
end


#=
Starting in the top left corner of a 2×2 grid, and only being able to move to the right and down, there are exactly 6 routes to the bottom right corner.
How many such routes are there through a 20×20 grid?
=#

# 1x1 = 2
# 2x2 = 6

# binomial coefficient is a way to calculate pascals triangle
# binomial(row, column)
# = row! / (column!*(row-column)!)
# reference: http://www.mathwords.com/b/binomial_coefficients_pascal.htm

function number_of_paths(x)
  x *= 2
  binomial(x,div(x,2))
end
@time println(number_of_paths(20))


"""
    CosineKernel

The cosine kernel is a stationary kernel for a sinusoidal given by
```
    κ(x,y) = cos( π * (x-y) )
```

"""
struct CosineKernel <: BaseKernel end

kappa(κ::CosineKernel, d::Real) = cospi(d)

metric(::CosineKernel) = Euclidean()


export isPowerOfTwo, isMersenneNumber, getMersenneNumber

function isPowerOfTwo(n::Integer)::Bool
    return (n != 0) && (n & (n - 1)) == 0
end

function isMersenneNumber(n::Integer)::Bool
    return isPowerOfTwo(n + 1)
end

function getMersenneNumber(n::Integer)::Integer
    return 2^n - 1
end


using LinearAlgebra
using SparseArrays
using ProximalOperators
using Test

@testset "LeastSquares" begin

@testset "$(T), $(s), $(matrix_type), $(mode)" for (T, s, matrix_type, mode) in Iterators.product(
    [Float64, ComplexF64],
    [(10, 29), (29, 10), (10, 29, 3), (29, 10, 3)],
    [:dense, :sparse],
    [:direct, :iterative],
)

if mode == :iterative && length(s) == 3
    # FIXME this case is currently not supported due to cg! in IterativeSolvers
    # See https://github.com/JuliaMath/IterativeSolvers.jl/issues/248
    # The fix is simple, but affects a lot of solvers in IterativeSolvers
    # Maybe we can use our own CG here and drop the dependency
    continue
end

R = real(T)

shape_A = s[1:2]
shape_b = if length(s) == 2 s[1] else s[[1, 3]] end
shape_x = if length(s) == 2 s[2] else s[2:3] end

A = if matrix_type == :sparse sparse(randn(T, shape_A...)) else randn(T, shape_A...) end
b = randn(T, shape_b...)
x = randn(T, shape_x...)

f = LeastSquares(A, b, iterative=(mode == :iterative))
predicates_test(f)

@test ProximalOperators.is_smooth(f) == true
@test ProximalOperators.is_quadratic(f) == true
@test ProximalOperators.is_generalized_quadratic(f) == true
@test ProximalOperators.is_set(f) == false

grad_fx, fx = gradient(f, x)
lsres = A*x - b
@test fx ≈ 0.5*norm(lsres)^2
@test all(grad_fx .≈ (A'*lsres))

call_test(f, x)
prox_test(f, x)
prox_test(f, x, R(1.5))

lam = R(0.1) + rand(R)
f = LeastSquares(A, b, lam, iterative=(mode == :iterative))
predicates_test(f)

grad_fx, fx = gradient(f, x)
@test fx ≈ (lam/2)*norm(lsres)^2
@test all(grad_fx .≈ lam*(A'*lsres))

call_test(f, x)
prox_test(f, x)
prox_test(f, x, R(2.1))

end

end