using DifferentialEquations
using Plots
using CPUTime

function main()
     a = 3.0  # パラメータ

     """
     微分方程式
     """
     dy(y, t, a) = a * y
     y0 = 1.0  # 初期値を設定
     tspan = (0.0, 1.0)  # 解く範囲を設定

     prob = ODEProblem(dy , y0 , tspan)
     sol = solve(prob)  # ソルバで解く

     # plot
     plot(
          sol,
          linewidth=5,
          xaxis="x",
          yaxis="y(x)",
          label="solution"
     )

     # savefig("solution_of_example_2.png")  #  保存
end

"""
オイラー法

・参考にしました -> [url](https://twitter.com/genkuroki/status/1301832571131633665/photo/1)
"""
function jisaku_solve_euler(dx, x₀, t_span, Δt)
     t = range(t_span..., step = Δt)  # 時間軸
     x = Vector{typeof(x₀)}(undef, length(t))  # 解を格納する1次元配列

     x[1] = x₀  # 初期値
     for i in 1:length(x)-1
          x[i+1] = x[i] + dx(t[i], x[i])Δt
     end

     t, x
end

"""ルンゲクッタ法（4次）"""
function jisaku_solve_RungeKutta(dx, x₀, t_span, Δt)
     t = range(t_span..., step = Δt)  # 時間軸
     x = Vector{typeof(x₀)}(undef, length(t))  # 解を格納する1次元配列

     x[1] = x₀  # 初期値
     for i in 1:length(x)-1
          k₁ = dx(t[i], x[i])
          k₂ = dx(t[i]+Δt/2, x[i]+k₁*Δt/2)
          k₃ = dx(t[i]+Δt/2, x[i]+k₂*Δt/2)
          k₄ = dx(t[i]+Δt, x[i]+k₃*Δt)
          x[i+1] = x[i] + (k₁ + 2k₂ + 2k₃ +k₄)Δt/6
     end

     t, x
end

function main()
     a = 3.0  # パラメータ
     dx(t, x) = a * x  # 微分方程式

     x₀ = 1.0
     t_span = (0.0, 5.0)
     Δt = 0.01
     t, x = jisaku_solve_RungeKutta(dx, x₀, t_span, Δt)
     plot(t, x, label="RungeKutta")

     t, x = jisaku_solve_euler(dx, x₀, t_span, Δt)
     plot!(t, x, label="euler")

     x = exp.(a*t)
     plot!(t, x, label="exact sol exp(at)")
end


"""Julia program to implement Subset Sum problem.
Given a set, we have to find if there exist a subset with the given sum.
The given problem can be solved using Dynamic Programming.
"""

function subset_sum(set, n, S)

    # Initialize the entire 2D matrix with 0 as value, signifying false.
    dp = zeros(Int, n + 1, S + 1)

    # When the sum is zero, the answer is always true, by forming an empty subset.
    for i in 1:n + 1
        dp[i,1] = 1
    end

    # When the given set is empty and the required sum is not zero, then such a subset cannot e formed.
    for i in 1:S + 1
        dp[1,i] = 0
    end

    """Tabulating the dp array in Bottom-Up manner. In such a way that each cell of the dp[i, j]
       array contains whether a subset of sum j can be formed with elements in the given set till i"""
    for i in 2:n + 1
        for j in 2:S + 1
            if j < (set[i - 1] + 1)
                dp[i,j] = dp[(i - 1),j]
            else
                if (dp[(i - 1),j] == 1)
                    dp[i, j] = 1
                elseif (dp[(i - 1),(j - set[i - 1])] == 1)
                    dp[i, j] = 1
                else
                    dp[i, j] = 0
                end
            end
        end
    end
    return dp[n + 1,S + 1]
end

print("What is the size of the set? ")
n = readline()
n = parse(Int, n)
arr = Int[]
if (n > 0)
    print("Enter the numbers: ")
    arr = [parse(Int, num) for num in split(readline())]
else
    print("The set is Empty!!!")
    exit()
end
print("Enter the sum? ")
S = readline()
S = parse(Int, S)
res = subset_sum(arr, n, S)
if (res == 1)
    print("A subset with the given sum is present in the array.")
else
    print("A subset with the given sum is not present in the array.")
end

"""
Time Complexity - O(n * S), where 'n' is the size of the array and 'S is the Sum
Space Complexity - O(n * S)

SAMPLE INPUT AND OUTPUT

SAMPLE I

What is the size of the set? 3
Enter the numbers: 3 2 1
Enter the sum? 55
A subset with the given sum is not present in the array.

SAMPLE II

What is the size of the set? 4
Enter the numbers: 10 20 30 40
Enter the sum? 60
A subset with the given sum is present in the array.
"""


export Linear

"""
**Linear function**

    Linear(c)

Returns the function
```math
f(x) = \\langle c, x \\rangle.
```
"""
struct Linear{R <: Real, A <: AbstractArray{R}} <: ProximableFunction
    c::A
end

is_separable(f::Linear) = true
is_convex(f::Linear) = true
is_smooth(f::Linear) = true

function (f::Linear{R})(x::AbstractArray{R}) where R
    return dot(f.c, x)
end

function prox!(y::AbstractArray{R}, f::Linear{R}, x::AbstractArray{R}, gamma::Union{R, AbstractArray{R}}=1.0) where R
    y .= x .- gamma.*(f.c)
    fy = dot(f.c, y)
    return fy
end

function gradient!(y::AbstractArray{R}, f::Linear{R}, x::AbstractArray{R}) where R
    y .= f.c
    return dot(f.c, x)
end

fun_name(f::Linear) = "Linear function"
fun_expr(f::Linear) = "x ↦ c'x"

function prox_naive(f::Linear, x, gamma)
    y = x - gamma.*(f.c)
    fy = dot(f.c, y)
    return y, fy
end


# Algorithm 2.5
# False Position method for finding roots
# Same as the secant method, but it checks
# if the roots are bracketed between successive
# iterations.

function false_approx(f, p0, p1, tolerance, maxIter)
	for i ∈ 2:maxIter
		q0 = f(p0)
		q1 = f(p1)
		p = p1 - q1*(p1 - p0)/(q1 - q0)
		if abs(p - p1) < tolerance
			return p
		end
		q = f(p)
		if q ⋅ q1 < 0
			p0 = p1
			q0 = q1
		end
		p1 = p
		q1 = q
	end
	throw(ArgumentError("Method failed"))
end

println(false_approx(x->(cos(x) - x), 0.5, π/4, 0.1, 100))


"""
    median(nums)

Finds median of a vector of numbers

## Example

```julia
median([2,1,3,4])                   # returns 2.5
median([2, 70, 6, 50, 20, 8, 4])    # returns 8
median([0])                         # returns 0
```

Contributed By:- [Ashwani Rathee](https://github.com/ashwani-rathee) and [Rratic](https://github.com/Rratic)
"""
function median(nums::Vector{T}) where {T<:Number}
    sorted = sort(nums)
    len = length(sorted)
    mid_index = div(len, 2)
    return if len % 2 == 0
        (sorted[mid_index+1] + sorted[mid_index]) / 2
    else
        sorted[mid_index+1]
    end
end


"""
rw_test(p; method = "VR", m = 1)

TEST OF RANDOM WALK

Random walk with uncorrelated increments (RW3) is the weakest form version of the random walk.
The other versions are random walk with iid increments (RW1) and random walk with independent increments (RW2).

Box and Pierce (1970) test of RW(3): the autocorrelation of increments at all leads and lags should be zero under random walk.
By summing the squared autocorrelation, the Box-Pierce Q-statistics is designed to detect departures from zero autocorrelations in either directions and at all lags

Ljung and Box (1978) test of RW(3): the Ljung-Box Q-statistics provides the finite-sample correction which yields a better fit to the chi-squared distribution for small sample size.

Variance ratio test of RW(1): the variance of random walk increments must be a linear function of the time interval under random walk. The variance ratios statistics tests such property.

INPUT
`p': a T x 1 vector of asset prices
`method': method used to test random walk hypthesis with uncorrelated increments:
        - BP: the Box and Pierce (1970) statistics
        - LB: the Ljung and Box (1978) statistics
        - VR: variance ratio test
`m': number of autocorrelation lags used for test

OUTPUT
`test_statistics': test statistics of the random walk test
`pvalue': p-value of the test statistics

"""

function rw_test(p, method = "VR", m = 1)
    r = diff(log.(p), dims = 1)
    T = length(r); # length of the time series

    if method == "BP" # Box and Pierce (1970) test of RW(3)
        Qm = 0.0;
        for k = 1:m
            rho_k = autocor(r, [k]; demean = false);
            Qm = Qm .+ rho_k.^2;
        end
        test_statistics = T * Qm;
        pvalue = 1.0 .- cdf.(Chisq(m), test_statistics);
    elseif method == "LB" # Ljung and Box (1978) test of RW(3)
        Qm = 0.0;
        for k = 1:m
            rho_k = autocor(r, [k]; demean = false);
            Qm = Qm .+ rho_k.^2;
        end
        test_statistics = T * (T + 2) / (T - m) * Qm;
        pvalue = 1.0 .- cdf.(Chisq(m), test_statistics);
    elseif method == "VR" # Variance ratio test of RW(1)
        VR = 1.0;
        for k = 1:m-1
            rho_k = autocor(r, [k]; demean = false);
            VR = VR .+ 2 .* (1 .- k / m) .* rho_k;
        end
        test_statistics = sqrt(T * m) * (VR .- 1.0) ./ sqrt(2.0 .* (m - 1));
        pvalue = 1.0 .- cdf.(Normal(0.0, 1.0), test_statistics);
    end

    return (test_statistics, pvalue)
end


"""
Julia program to check if a number is a Strong number or not.
Strong Number is a number whose sum of the factorial of digits
is equal to the number itself.
"""

# Helper function to return the factorial of a number
function factorial(num)
    if ( num == 1 || num == 0 )
        return 1
    end
    return num * factorial( num - 1)
end

""" Extract each digit of the given number and calculate the sum of the
    factorial of each digit present in the given number"""
function strong_number(num)
    temp = num
    sum = 0
    while(temp > 0)
        rem = temp % 10
        sum = sum + factorial(rem)
        temp = temp ÷ 10
    end
    return sum
end

print("Enter the number: ")
num = readline()
num = parse(Int, num)
sum = strong_number(num)
if sum == num
    println("The given number $num is a Strong Number.")
else
    println("The given number $num is not a Strong Number.")
end

"""
Time Complexity: O(n*log(n)), where 'n' is the given number
Space Complexity: O(1)

SAMPLE INPUT AND OUTPUT

SAMPLE 1
Enter the number: 3124
The given number 3124 is not a Strong Number.

SAMPLE 2
Enter the number: 145
The given number 145 is a Strong Number.
"""


### A Pluto.jl notebook ###
# v0.16.0

using Markdown
using InteractiveUtils

# ╔═╡ cc106912-0b14-4981-a67b-580dda8a56ed
begin
	using Pkg
	cd(joinpath(dirname(@__FILE__),".."))
    Pkg.activate(pwd())
    using Distributions, LinearAlgebra, Plots, InteractiveUtils
end

# ╔═╡ 235e2200-fce9-11ea-0696-d36cddaa843e
md"""
# Phase transition in networks
## Generalities - Networks

A network is a set of objects (called nodes or vertices) that are connected together. The connections between the nodes are called edges or links. In mathematics, networks are often referred to as [graphs](https://en.wikipedia.org/wiki/Graph_(discrete_mathematics)).

In the following, we will only consider undirected networks i.e. if node $i$ is connected to node $j$, then node $j$ is automatically connected to node $i$ (as is the case with Facebook friendships). For unweighted networks of $N$ nodes, the network structure can be represented by an $N \times N$ adjacency matrix $A$:

```math
a_{i,j} = \left\{
                \begin{array}{ll}
                  1 & \text{if there is an edge from node $i$ to node $j$}\\
                  0 & \text{otherwise}\\
                \end{array}
                \right.
```
The degree of a node $i$ is the number of connections it has. In terms of the adjacency matrix
$A$, the degree of node $i$ is the sum of the i$^{\text{th}}$ row of $A$:
$$k_i = \sum_{j=1}^{N}a_{i,j}$$

The average node degree $\langle k \rangle$ is then given by: $$\langle k \rangle = \sum_{i=1}^{N}k_{i}$$

## Erdös-Rényi random graph model
One of the simplest graph models is the Erdös-Rényi random graph model, denoted by $\mathcal{G}(N,p)$ with $N$ being the amount of nodes and $p$ being the probability that a link exists between two nodes. Self-loops are excluded. The value of $p$ is typically small (this is to avoid that the average degree $\langle k \rangle$ depends on $N$, cf. specialised literature for more details). When studying random graph model models, a major aim is to predict the average behaviour of certain network metrics and, if possible, their variance.

The Erdös-Rényi random graph  exhibits a phase transition. Let us consider the size (i.e., number of nodes) of the largest connected component in the network as a function of the mean degree ⟨k⟩. When ⟨k⟩ = 0, the network is trivially composed of N disconnected nodes. In the other extreme of ⟨k⟩ = N − 1, each node pair is adjacent such that the network is trivially connected. Between the two extremes, the network does not change smoothly in terms of the largest component size. Instead, a giant component, i.e., a component whose size is the largest and proportional to N, suddenly appears as ⟨k⟩ increases, marking a phase transition. The goal of this application is to determine this value by simulation.

## Problem solution
We split the problem in a series of subproblems:
* generating a random graph
* determine the average degree
* identifying the size of the largest connected component
* visualising the result
* determine the critical value

"""

# ╔═╡ 976bb1e0-07c9-11eb-2a4f-fb7aac69f8ec
begin
	function gengraph(N::Int, p::Float64)
		d = Bernoulli(p) # define the distribution
		A = rand(d, N, N)
		# set main diagonal to 0
		A[diagind(A)] = zeros(Int,N)
		return LinearAlgebra.Symmetric(A)
	end

	function avgdegree(A)
		return mean(sum(A, dims=1))
	end

	"""
		neighbors(A, i)

	Détermine les voisins du noeud i dans le graph représenté par A
	"""
	function neighbors(A, i)
		findall(x -> isequal(x, 1), @view A[i,:])
	end

	function components(A)
		N = size(A,1) # network size
		visited = Dict(i => false for i in 1:N)
		comps = []
		for n in 1:N
			if !visited[n]
				push!(comps,dig(A, n, visited))
				end
		end

		return maximum(length,comps) / N
	end

	function dig(A, n, visited, comp=[])
		visited[n] = true
		push!(comp, n)
		for neigbor in neighbors(A, n)
			if !visited[neigbor]
				dig(A, neigbor, visited, comp)
			end
		end
		return comp
	end

	function sim(N::Int, p::Float64)
		A = gengraph(N,p)
		return components(A), avgdegree(A)
	end
end

# ╔═╡ 42f5cca6-07d1-11eb-3169-cd3650c42081
N = 1000

# ╔═╡ a060dc62-07cb-11eb-2aaa-7190bde5dc36
# simulation part (range of probabilities, 10 simulations for each probability)
let N = N
	P = range(5e-5, 3e-3, length=20)
	rat_comp = Array{Float64, 1}()
	k_moy = Array{Float64, 1}()
	for p in P
		for _ in 1:10
			res = sim(N,p)
			push!(rat_comp, res[1])
			push!(k_moy, res[2])
		end
	end
	scatter(k_moy, rat_comp, ylims=(0,1), label="", xlabel="<k>", ylabel="N_largest_comp / N")
end

# ╔═╡ 1f0d3e38-402b-4d82-a041-c623c5d531b3
md"""
# Test from previous years
* ants foraging (cf. "./Exercises/data/test2019.pdf")
* housing evolution (cf. "./Exercises/data/test2020.pdf")

"""

# ╔═╡ Cell order:
# ╠═cc106912-0b14-4981-a67b-580dda8a56ed
# ╟─235e2200-fce9-11ea-0696-d36cddaa843e
# ╠═976bb1e0-07c9-11eb-2a4f-fb7aac69f8ec
# ╠═42f5cca6-07d1-11eb-3169-cd3650c42081
# ╠═a060dc62-07cb-11eb-2aaa-7190bde5dc36
# ╟─1f0d3e38-402b-4d82-a041-c623c5d531b3


# # Poisson Equation
#
# We want to solve the [poisson equation](https://en.wikipedia.org/wiki/Poisson_equation) on the unit interval. It is given by
# `Δu = f` with boundary conditions `u(0) = a` and `u(1) = b`
# First of all let us choose some values for the parameters and remark, that there is an exact solution:
f = 1.0
a = -1.0
b = 2.0

u_analytic(x) = f/2*x^2 + (b-a-f/2) * x + a

# We would like to recompute this solution numerically
using DiffEqOperators

nknots = 10
h = 1.0/(nknots+1)
ord_deriv = 2
ord_approx = 2

Δ = CenteredDifference(ord_deriv, ord_approx, h, nknots)
bc = DirichletBC(a, b)

# Before solving the equation, lets take a look at Δ and bc:
# display(Array(Δ))
# display(bc*zeros(nknots))
# We see that `Δ` is a (lazy) matrix with the laplace stencil extended over the boundaries.
# And `bc` acts by padding the values just outside the boundaries.

u = (Δ*bc) \ fill(f, nknots)
knots = range(h, step=h, length=nknots)

# Since we used a second order approximation and the analytic solution itself was a second order
# polynomial, we expect that they are equal up to rounding errors:
using Test
@test u ≈ u_analytic.(knots)


using NewtonsMethod
using Test

@testset "NewtonsMethod.jl" begin

    # Define some functions to be tested

    # a. root = 1.0
    f(x) = (x - 1)^3
    df(x) = 3*(x - 1)^2

    # b. root = 0.6
    g(x) = (5*x - 3)^2
    dg(x) = 10*(5*x - 3)

    # c. no root (root = nothing)
    h(x) = 2 + x^2
    dh(x) = 2*x

    # 1. Test function f(x)
    @test NewtonsMethod.newtonroot(f, df)[1] ≈ 1.0
    @test NewtonsMethod.newtonroot(f)[1] ≈ 1.0

    # 2. Test function g(x)
    @test NewtonsMethod.newtonroot(g, dg)[1] ≈ 0.6
    @test NewtonsMethod.newtonroot(g)[1] ≈ 0.6

    # 3. Test using BigFloat precision instead of Float
    @test BigFloat(NewtonsMethod.newtonroot(f, df)[1]) ≈ 1.0
    @test BigFloat(NewtonsMethod.newtonroot(f)[1]) ≈ 1.0

    x0 = BigFloat(0.3)
    @test NewtonsMethod.newtonroot(f, df; iv = x0)[1] ≈ 1.0
    @test NewtonsMethod.newtonroot(f; iv = x0)[1] ≈ 1.0

    # 4. Test non-convergence of h(x)
    @test NewtonsMethod.newtonroot(h, dh)[1] === nothing
    @test NewtonsMethod.newtonroot(h)[1] === nothing

    # 5. Test maxiter
    # From PS1, root of f(x) found in ≈ 20 iterations
    @test NewtonsMethod.newtonroot(f, df; maxiter = 1)[1] === nothing
    @test NewtonsMethod.newtonroot(f; maxiter = 1)[1] === nothing
    @test NewtonsMethod.newtonroot(f, df; maxiter = 200)[1] ≈ 1.0
    @test NewtonsMethod.newtonroot(f; maxiter = 200)[1] ≈ 1.0

    # 6. Test tolerance
    @test !(NewtonsMethod.newtonroot(f, df; tol = 0.1)[1] ≈ 1.0)
    @test !(NewtonsMethod.newtonroot(f; tol = 0.1)[1] ≈ 1.0)

    # 7. Test initial value
    @test NewtonsMethod.newtonroot(f, df; iv = 0.5)[1] ≈ 1.0
    @test NewtonsMethod.newtonroot(f; iv = 0.5)[1] ≈ 1.0

    # 8. Break a test
    # @test NewtonsMethod.newtonroot(f, df)[1] ≈ 0.2
    # @test NewtonsMethod.newtonroot(f)[1] ≈ 0.2

end


# ------------------------------------------------------------------------------------------
# # Entropies & Dimensions
#
#
# Topics:
# * Entropies from **DynamicalSystems.jl**
# * Generalized Dimension
# * Automated dimension estimation!
# * Other related concepts
# * Docstrings
#
# ---
#
# # Generalized Entropy
#
# * In the study of dynamical systems there are many quantities that identify as "entropy".
# * These quantities are not the more commonly known [thermodynamic
# ones](https://en.wikipedia.org/wiki/Entropy), used in Statistical Physics.
# * Rather, they are more like the entropies of [information
# theory](https://en.wikipedia.org/wiki/Entropy_(information_theory), which represent
# information contained within a dataset.
# * In general, the more "uncertain" or "random" the dataset is, the larger its entropy will
# be. On the other hand, the lower the entropy, the more "predictable" the dataset becomes.
#
#
# Let $p$ be an array of probabilities (such that it sums to 1). Then the generalized
# entropy is defined as
#
# $$
# R_\alpha(p) = \frac{1}{1-\alpha}\log\left(\sum_i p[i]^\alpha\right)
# $$
#
# and is also called [Rényi entropy](https://en.wikipedia.org/wiki/R%C3%A9nyi_entropy).
# Other entropies, like e.g. the [Shannon
# entropy](https://en.wikipedia.org/wiki/Entropy_(information_theory) are generalized by it,
# since for $\alpha = 1$, the Rényi entropy becomes the Shannon entropy,
#
# $$
# R_1(p) = -\left(\sum_i p[i] \log (p[i]) \right)
# $$
#
# The Rényi entropy can be computed for a specific dataset, given $p$. But how does one get
# $p$?
# 1. $p$ represents the probability that a point of a dataset falls into a specific "bin".
# 2. It is nothing more than the (normalized) histogram of the dataset!
# ------------------------------------------------------------------------------------------

using DynamicalSystems

# ------------------------------------------------------------------------------------------
# Let's generate a dataset so that we can practice calculating entropies.
# ------------------------------------------------------------------------------------------

N = 100000
randomdata = Dataset(rand(N,3))

# ------------------------------------------------------------------------------------------
# ---
#
# ```julia
# genentropy(α, ε, dataset::AbstractDataset; base = e)
# ```
# * This function calculates the generalized entropy of order `α`.
# * It first calculates the probability array $p$.
# * The "histogram" is created by partitioning the `dataset` into boxes of size `ε`.
#
# ------------------------------------------------------------------------------------------

genentropy(2, 0.1, randomdata)

genentropy(2, 0.01, randomdata)

genentropy(2, 0.001, randomdata)

genentropy(2, 0.0001, randomdata)

# ------------------------------------------------------------------------------------------
# Note that the output of `genentropy` changed with changing $\varepsilon$ until we hit
# $\varepsilon = 0.001$.
#
# At this point the value for the entropy has already saturated. There's no use in
# partitioning the dataset in smaller boxes.
#
# ---
#
# Now let's calculate the entropy of a coin toss!
#
# First, let's create an array, `y`, that stores the results of coin tosses as `0`s or `1`s
# for `1000000` tosses.
# ------------------------------------------------------------------------------------------

y = Float64.(rand(Bool, 1000000))

sh = genentropy(1, 0.1, y) # Renyi entropy with α = 1 is the Shannon entropy

# ------------------------------------------------------------------------------------------
# The above number should be log(2) [by
# definition](https://en.wikipedia.org/wiki/Shannon_(unit)
# ------------------------------------------------------------------------------------------

isapprox(sh, log(2), rtol = 1e-4)

# ------------------------------------------------------------------------------------------
# `genentropy` is conveniently used with `trajectory` outputs.
#
# Here we create a trajectory for a towel map,
# ------------------------------------------------------------------------------------------

towel = Systems.towel()
tr = trajectory(towel, N-1);
summary(tr)

# ------------------------------------------------------------------------------------------
# and calculate its entropy:
# ------------------------------------------------------------------------------------------

genentropy(1, 0.01, tr) # The result is with log base-e !

# ------------------------------------------------------------------------------------------
# Let's also compare the entropy of the above dataset (a trajectory of the towel map) with
# that of a random dataset:
# ------------------------------------------------------------------------------------------

genentropy(1, 0.01, randomdata)

# ------------------------------------------------------------------------------------------
# * As expected, the entropy of the random dataset is higher.
#
# ---
#
# How much time does the computation take?
# ------------------------------------------------------------------------------------------

using BenchmarkTools
@btime genentropy(1, 0.01, $tr);

# ------------------------------------------------------------------------------------------
# ## Specialized histogram
# * Partitioning the dataset (i.e. generating a "histogram") is in general a costly
# operation that depends exponentially on the number of dimensions.
# * In this specific application however, we can tremendously reduce the memory allocation
# and time spent!
#
# Let's get the array of probabilities $p$ for size ε where `tr` is the trajectory of the
# towel map
# ------------------------------------------------------------------------------------------

ε = 0.01
p = non0hist(ε, tr)

# ------------------------------------------------------------------------------------------
# Here's a sanity check, showing our probabilities should sum to roughly `1`.
# ------------------------------------------------------------------------------------------

sum(p)

# ------------------------------------------------------------------------------------------
# How long does generating the histogram take?
# ------------------------------------------------------------------------------------------

@btime non0hist($ε, $tr);

# ------------------------------------------------------------------------------------------
# How long does this take if we create 9-dimensional data and compare again?
# ------------------------------------------------------------------------------------------

nine = Dataset(rand(N, 9))
@btime non0hist($ε, $nine);

# ------------------------------------------------------------------------------------------
# * We went from dimension 3 to dimension 9 but the time roughly only tripled
# ------------------------------------------------------------------------------------------

# ------------------------------------------------------------------------------------------
# # Attractor Dimension
# 1. There are numerous methods that one can use to calculate a so-called "dimension" of a
# dataset, like for example the [Fractal
# dimension](https://en.wikipedia.org/wiki/Fractal_dimension).
#
# 2. Most of the time these dimensions indicate some kind of scaling behavior.
#
# 3. For example, the scaling of `genentropy` with decreasing `ε` gives the so-called
# "generalized dimension".
#
#
# $ E \approx -D\log(\varepsilon)$ with $E$ the entropy and $D$ the "dimension".
#
# ---
# I want to know dimension of attractor of the Towel Map!
#
#
# ------------------------------------------------------------------------------------------

towel = Systems.towel()
towel_tr = trajectory(towel, 1000000);
summary(towel_tr)

# ------------------------------------------------------------------------------------------
# Note that more points = more precision = more computations = more time!
#
# Now I want to compute `genentropy` for different ε.
#
# Which ε should we use...?
#
# Let's do a "random" logspace based guess...
# ------------------------------------------------------------------------------------------

ες = logspace(-4, 1, 12)

Es = zeros(ες)
for (i, ε) ∈ enumerate(ες)
    Es[i] = genentropy(1, ε, towel_tr)
end
Es

# ------------------------------------------------------------------------------------------
# **Shorter version:**
# ------------------------------------------------------------------------------------------

Es = genentropy.(1, ες, towel_tr)

# ------------------------------------------------------------------------------------------
# Alright. Remember that it should be that $E \approx -D\log(\varepsilon)$
#  with $E$ the entropy and $D$ the "dimension".
#
# Let's plot and see:
# ------------------------------------------------------------------------------------------

using PyPlot; figure(figsize = (6,4))
x = -log.(ες)
plot(x, Es); xlabel("-log(ε)"); ylabel("Entropy");
plot([x[4], x[4]], [0, 15], color = "C1", alpha = 0.5)
plot([x[end-3], x[end-3]], [0, 15], color = "C1", alpha = 0.5);

# ------------------------------------------------------------------------------------------
# What typically happens is that there is some region where this scaling behavior holds, but
# then it stops holding due to the finite amount of data points.
#
# Above, the expected scaling behavior holds between the orange vertical lines.
#
# Let's choose the curve points that do fall in the linear regime of the above plot,
# ------------------------------------------------------------------------------------------

x, y = -log.(ες)[4:end-2], Es[4:end-2]

# ------------------------------------------------------------------------------------------
# and find the slope of the curve there, to calculate the dimension, D.
# ------------------------------------------------------------------------------------------

offset, slope = linreg(x, y)
D = slope

# ------------------------------------------------------------------------------------------
# This is actually a correct result, the information dimension of the attractor of the towel
# map is around 2.
#
# ---
#
# * Are the values of `ες` we used good?
# * For a general dataset, how can we determine them?
#
# the function `estimate_boxsizes(dataset; kwargs...)` can help with that!
# ------------------------------------------------------------------------------------------

ες = estimate_boxsizes(towel_tr)

# ------------------------------------------------------------------------------------------
# Let's plot $E$ vs. $-\log \epsilon$ again
# ------------------------------------------------------------------------------------------

Es = genentropy.(1, ες, towel_tr)
figure(figsize = (6,4))
plot(-log.(ες), Es); xlabel("-log(ε)"); ylabel("E");

# ------------------------------------------------------------------------------------------
# ---
# # Automated Dimension Estimation
#
# Given some arbitrary plot like the one above, is there any algorithm to deduce a scaling
# region???
#
# The function `linear_regions(x, y; kwargs...)` decomposes the function `y(x)` into regions
# where  the function is linear.
#
# It returns the indices of `x` that correspond to linear regions and the approximated
# tangents at each region!
# ------------------------------------------------------------------------------------------

x = -log.(ες)
lrs, slopes = linear_regions(x, Es)

for i in 1:length(slopes)
    println("linear region $(i) starts from index $(lrs[i]) and ends at index $(lrs[i+1])")
    println("with corresponding slope $(slopes[i])")
    println()
end

# ------------------------------------------------------------------------------------------
# The linear region which is biggest is "probably correct one".
# Here the last linear region is the largest; thus the slope is
# ------------------------------------------------------------------------------------------

slopes[end]

# ------------------------------------------------------------------------------------------
# This `linear_regions` function seems very shady... Is there any "easy" way to visualize
# what it does? *Say no more!*
#
# In this next example, we'll use `ChaosTools`, which belows to `DynamicalSystems`.
# ------------------------------------------------------------------------------------------

using PyPlot, ChaosTools

# ------------------------------------------------------------------------------------------
# `ChaosTools` provides a function, `plot_linear_regions` !
# ------------------------------------------------------------------------------------------

figure(figsize=(6,4))
plot_linear_regions(x, Es)
xlabel("-log(ε)"); ylabel("E");

# ------------------------------------------------------------------------------------------
# Adjust the tolerance of `linear_regions` using keyword argument `tol`
# ------------------------------------------------------------------------------------------

figure(figsize=(6,4))
plot_linear_regions(x, Es; tol = 0.8)
xlabel("-log(ε)"); ylabel("E");

# ------------------------------------------------------------------------------------------
# Notice how the color schemes allow us to visualize that adjusting the tolerance reduces
# our number of linear regimes!
# ------------------------------------------------------------------------------------------

# ------------------------------------------------------------------------------------------
# ## `generalized_dim` function
# ------------------------------------------------------------------------------------------

# ------------------------------------------------------------------------------------------
# Let's summarize what we just did to estimate the dimension of an attractor.
#
# 1. We decided on some partition sizes `ες` to use (the function `estimate_boxsizes` can
# give an estimate for that).
# 2. For each `ε` in `ες` we calculated the entropy via `genentropy`. We stored these
# entropies in an array `Es`.
# 3. We tried to find a "linear scaling region" of the curve `Es` vs. `-log.(ες)`.
# 4. The slope of this "linear scaling region" is the dimension we estimated.
#
# Wouldn't it be **cool** if all of this process could happen with one function call?
#
# This is *exactly* what the following function does:
# ```julia
# generalized_dim(α, dataset, ες = estimate_boxsizes(tr))
# ```
# which computes the `α`-order generalized dimension.
# ------------------------------------------------------------------------------------------

generalized_dim(2.0, tr)

generalized_dim(1.0, tr)

# ------------------------------------------------------------------------------------------
# The first input to `generalized_dim` is our value for $\alpha$!
# ------------------------------------------------------------------------------------------

# ------------------------------------------------------------------------------------------
# Similarly, let's calculate the dimension of the Henon map that we have seen in previous
# tutorials,
# ------------------------------------------------------------------------------------------

hen = Systems.henon()
tr = trajectory(hen, 200000)
generalized_dim(0, tr)

# ------------------------------------------------------------------------------------------
# and of the Lorenz system:
# ------------------------------------------------------------------------------------------

lor = Systems.lorenz()
tr_lor = trajectory(lor, 1000.0; dt = 0.05);

generalized_dim(2.0, tr_lor)

# ------------------------------------------------------------------------------------------
# The correlation dimension of the Lorenz attractor (for default parameters) is reported
# *somewhere* around `2.0` (Grassberger and Procaccia, 1983).
#
# ## `generalized_dim` is but a crude estimate!
#
# **It is important to understand that `generalized_dim` is only a crude estimate! You must
# check and double-check and triple-check if you want more accuracy!**
# ------------------------------------------------------------------------------------------

# ------------------------------------------------------------------------------------------
# # Other related concepts
# ## Kaplan-Yorke dimension
# The Kaplan-Yorke dimension is defined as simply the (interpolated) number where the sum of
# Lyapunov exponents crosses zero.
#
# This simple interpolation is done by the function `kaplanyorke_dim(ls)`.
# ------------------------------------------------------------------------------------------

lor = Systems.lorenz()

ls = lyapunovs(lor, 4000.0; Ttr = 100.0)

kaplanyorke_dim(ls)

# ------------------------------------------------------------------------------------------
# ## Permutation Entropy
#
# Another entropy-like quantity that you can compute with **DynamicalSystems.jl** is the
# [permutation entropy](https://juliadynamics.github.io/DynamicalSystems.jl/latest/chaos/ent
# ropies/#permutation-entropy).
#
# This is done with the function `permentropy(s, order; ...)`. The permutation entropy is
# different because it requires a *timeseries* `s` as the input.
# ------------------------------------------------------------------------------------------

# create timeseries:
s = (ds = Systems.towel(); trajectory(towel, 10000)[:, 1])
# order `o` permutation entropy:
o = 6
permentropy(s, o)

# ------------------------------------------------------------------------------------------
# # Docstrings
# ------------------------------------------------------------------------------------------

?genentropy

?generalized_dim

?non0hist

?estimate_boxsizes

?linear_regions

?kaplanyorke_dim

?permentropy


### A Pluto.jl notebook ###
# v0.12.16

using Markdown
using InteractiveUtils

# ╔═╡ e077ab16-22e5-11eb-15ed-5f0654bf7328
using Pkg, DrWatson

# ╔═╡ e75a9a06-22e5-11eb-2e36-8d4f62b830ed
begin
	@quickactivate "StatisticsWithJuliaPlutoNotebooks"
	using Random, Distributions, Plots, LaTeXStrings
	Random.seed!(0)
end

# ╔═╡ ca5fbdc8-22e5-11eb-0a60-cb0a127a3ca5
md"## Listing5.09"

# ╔═╡ fa68607e-22e5-11eb-0558-c9a4d9f77426
begin
	actualAlpha, actualLambda = 2,3
	gammaDist = Gamma(actualAlpha,1/actualLambda)
	n = 10^2
	sample = rand(gammaDist, n)
end

# ╔═╡ 68b12c46-3a9c-11eb-03b3-bf9862c5371c
begin
	alphaGrid = 1:0.02:3
	lambdaGrid = 2:0.02:5
end

# ╔═╡ 68b16da0-3a9c-11eb-21e2-53c689f1599c
likelihood = [prod([pdf.(Gamma(a,1/l),v) for v in sample])
                        for l in lambdaGrid, a in alphaGrid]

# ╔═╡ 68b1f8ce-3a9c-11eb-2ad9-a5af4bd66c0e
surface(alphaGrid, lambdaGrid, likelihood, lw=0.1,
	c=cgrad([:blue, :red]), legend=:none, camera = (135,20),
	xlabel=L"\alpha", ylabel=L"\lambda", zlabel="Likelihood")

# ╔═╡ 475ff888-22e6-11eb-2354-09f8f40a8e12
md"## End of listing5.09"

# ╔═╡ Cell order:
# ╟─ca5fbdc8-22e5-11eb-0a60-cb0a127a3ca5
# ╠═e077ab16-22e5-11eb-15ed-5f0654bf7328
# ╠═e75a9a06-22e5-11eb-2e36-8d4f62b830ed
# ╠═fa68607e-22e5-11eb-0558-c9a4d9f77426
# ╠═68b12c46-3a9c-11eb-03b3-bf9862c5371c
# ╠═68b16da0-3a9c-11eb-21e2-53c689f1599c
# ╠═68b1f8ce-3a9c-11eb-2ad9-a5af4bd66c0e
# ╟─475ff888-22e6-11eb-2354-09f8f40a8e12


include("ceil_floor.jl")

"""
    median(nums)

Finds median of a vector of numbers

# Example

```julia
median([2,1,3,4])                   # returns 2.5
median([2, 70, 6, 50, 20, 8, 4])    # returns 8
median([0])                         # returns 0
```

Contributed By:- [Ashwani Rathee](https://github.com/ashwani-rathee)
"""
function median(nums)
    sorted = sort(nums)
    len = length(sorted)
    mid_index = Int(floor_val(len / 2))
    return len % 2 == 0 ? (sorted[mid_index+1] + sorted[mid_index]) / 2 : sorted[mid_index+1]
end


using DiffEqBiological
using DifferentialEquations
using Plots
using Colors

logistic_model = @reaction_network logistic begin
  r, cell + nutrient --> 2cell
end r

r = 1.5
cell0 = 1.0
nutrient0 = 2000.0

tspan = (0.0,0.008)
p =  (r)
u0 = [cell0, nutrient0]

oprob = ODEProblem(logistic_model, u0, tspan, p)
osol = solve(oprob, reltol=1e-8,abstol=1e-8)
Kbio = nutrient0+cell0
n = [((Kbio)*(cell0*exp(r*t)))/((Kbio)+(cell0*(exp(r*t)-1))) for t in osol.t]
#plot(osol)
plot(osol.t, [osol.u[i][1] for i in 1:length(osol.t)], labels="Cells", color= "blue", lw=2.0)
plot!(osol.t, [osol.u[i][2] for i in 1:length(osol.t)], labels="Nutrients", color="red", lw=2.0)

# https://nextjournal.com/sosiris-de/diffeqbiological
# What is the final value of cell(t)?  How does it relate to K?
# Can you overlay the analytical solution?

prob = DiscreteProblem(u0, tspan ,p)
jump_prob = JumpProblem(prob,Direct(),logistic_model)
sol = solve(jump_prob,FunctionMap())
#plot(sol)
plot(sol.t, [sol.u[i][1] for i in 1:length(sol.t)], labels="", color="blue", lw=2)
plot!(sol.t, [sol.u[i][2] for i in 1:length(sol.t)], labels="", color="red", lw=2)
# Can you plot the stochastic solution (sol) on top of the deterministic solution (osol)?

plot(osol)
plot!(sol)

#or

p1=plot(osol)
p2=plot(sol)

plot(p1,p2, xaxis="Time", yaxis="Population Size")

# Can you see how step-like the results are?
# What happens if you increase nutrient0 to 2000.0?
# Do the stochastic results still look very different to the deterministic ones?
# If you run line 27 above several times, saving the results in a list comprehension,
# then drawing them all on the same plot, will they be the same?

nsins=100
prob = DiscreteProblem(u0, tspan ,p)
jump_prob = JumpProblem(prob,Direct(),logistic_model)
solutions =[solve(jump_prob,FunctionMap()) for i in 1:nsins]
plot(solutions[1].t, [solutions[1].u[i][1] for i in 1:length(solutions[1].t)], labels="Cells", color= "blue", xaxis="Time", yaxis="Population Size", title="Discrete Stochastic Populations Dynamics", lw=0.3)
plot!(solutions[1].t,[solutions[1].u[i][2] for i in 1:length(solutions[1].t)], labels="Nutrients", color="red", lw=0.3)
for j in 2:nsins
    p=plot!(solutions[j].t, [solutions[j].u[i][1] for i in 1:length(solutions[j].t)], labels="", color= "blue", lw=0.3)
    q=plot!(solutions[j].t,[solutions[j].u[i][2] for i in 1:length(solutions[j].t)], labels="", color="red", lw=0.3)
    display(p)
    display(q)
end
oprob = ODEProblem(logistic_model, u0, tspan, p)
osol = solve(oprob, reltol=1e-8,abstol=1e-8)
Kbio = nutrient0+cell0
n = [((Kbio)*(cell0*exp(r*t)))/((Kbio)+(cell0*(exp(r*t)-1))) for t in osol.t]
plot!(osol.t, [osol.u[i][1] for i in 1:length(osol.t)], labels="Deterministic Continuous Solution", color="black", lw=2)
plot!(osol.t, [osol.u[i][2] for i in 1:length(osol.t)], labels="", color="black", legend=:right, lw=2)

# Can you simulate from the discrete stochastic exponential model?

############################################################################################################

using DiffEqBiological
using DifferentialEquations
using Plots
using Colors

exponential_model = @reaction_network exponential begin
  r, cell  --> 2cell
end r

r = 5.0
cell0 = 10.0

tspan = (0.0,1.0)
p = (r)
u0 = [cell0]

oprobe = ODEProblem(exponential_model, u0, tspan, p)
osole = solve(oprobe)
plot(osole, labels="Deterministic Solution", lw=2)

probe = DiscreteProblem(u0, tspan ,p)
jump_prob = JumpProblem(probe,Direct(),exponential_model)
sole = solve(jump_prob,FunctionMap())
#plot(sole)
plot!(sole, title="Deterministic and Stochastic Exponential Model", xaxis="Time", yaxis="Popuplation Size", legend=:left, lw=2, labels="Stochastic Solution")

nsims=100
probe = DiscreteProblem(u0, tspan ,p)
jump_prob = JumpProblem(probe,Direct(),exponential_model)
solutions =[solve(jump_prob,FunctionMap()) for i in 1:nsims]
plot(solutions[1].t, [solutions[1].u[i][1] for i in 1:length(solutions[1].t)], labels="Cells", color="blue", lw=0.2)
for j in 2:nsims
    p=plot!(solutions[j].t, [solutions[j].u[i][1] for i in 1:length(solutions[j].t)], labels="", color="blue", legend=:left, title="Exponential Model w/ Stochastic and Deterministic Solutions", lw=0.2)
    display(p)
end
oprobe = ODEProblem(exponential_model, u0, tspan, p)
osole = solve(oprobe)
plot!(osole, labels="Deterministic Solution", xaxis="Time", yaxis="Population Size",color="black", lw=2)

######################################################################################################################

# Can you simulate from the discrete stochastic Lotka-Volterra model?

lv_model = @reaction_network lv begin
  k1, prey --> 2prey
  k2, prey + predator --> 2predator
  k3, predator --> null
 # k4, prey --> null
end k1 k2 k3 #k4

k1 = 1.0
k2 = 0.1
k3 = 0.1
#k4 = 0.001

prey0 = 40.0
predator0 = 100.0
null0 = 0.0
tspan = (0.0,200.0)
p =  (k1,k2,k3)
u0 = [prey0, predator0]

oproblv = ODEProblem(lv_model, u0, tspan, p)
osollv = solve(oproblv, reltol=1e-5,abstol=1e-5)
#plot(osollv)
plot(osollv.t, [osollv.u[i][1] for i in 1:length(osollv.t)], labels="D-Prey", color= "blue", lw=2)
plot!(osollv.t, [osollv.u[i][2] for i in 1:length(osollv.t)], labels="D-Predators", color="orange", lw=2)

prob = DiscreteProblem(u0, tspan ,p)
jump_prob = JumpProblem(prob,Direct(),lv_model)
sollv = solve(jump_prob,FunctionMap())
#plot(sollv)
plot!(sollv.t, [sollv.u[i][1] for i in 1:length(sollv.t)], labels="S-Prey", color= "green", lw=2.0)
plot!(sollv.t, [sollv.u[i][2] for i in 1:length(sollv.t)], labels="S-Predators", color="red", lw=2.0, title="Lotka-Volterra Deterministic and Stochastic Models", xaxis="Time", yaxis="Population Size")

# at larger pops the stochastic solution looks very similar to the deterministic solution but it doesnt have the advantage of being continuous

# Can you simulate from the discrete stochastic birth-death model?

############################################################################################################################

bd_model = @reaction_network bd begin
  lambda, cell  --> 2cell
  mu, cell  --> null
end lambda mu

lambda = 2.0
#lambda = 0.8
mu= 1.5
#mu= 0.5
cell0 = 100.0
null0 = 0.0

tspan = (0.0,5.0)
p =  (lambda, mu)
u0 = [cell0, null0]

oprobbd = ODEProblem(bd_model, u0, tspan, p)
osolbd = solve(oprobbd, reltol=1e-8,abstol=1e-8)
#plot(osolbd)
plot(osolbd.t, [osolbd.u[i][1] for i in 1:length(osolbd.t)], labels="", color= "blue", ls=:dash)
plot!(osolbd.t, [osolbd.u[i][2] for i in 1:length(osolbd.t)], labels="", color="red", ls=:dash)

probbd = DiscreteProblem(u0, tspan ,p)
jump_prob = JumpProblem(probbd,Direct(),bd_model)
solbd = solve(jump_prob,FunctionMap())
#plot(solbd)
plot!(solbd.t, [solbd.u[i][1] for i in 1:length(solbd.t)], labels="Cells", color= "blue")
plot!(solbd.t, [solbd.u[i][2] for i in 1:length(solbd.t)], labels="Dead Cells", color="red", legend=:left, title="Birth-Death Model w/ Deterministic and Stochastic Solutions", xaxis="Time", yaxis="Popuplation Size")

nsins=100
probbd = DiscreteProblem(u0, tspan ,p)
jump_prob = JumpProblem(probbd,Direct(),bd_model)
solutions =[solve(jump_prob,FunctionMap()) for i in 1:nsins]
plot(solutions[1].t, [solutions[1].u[i][1] for i in 1:length(solutions[1].t)], labels="Cells", color= "blue", xaxis="Time", yaxis="Population Size", title="Discrete Stochastic Populations Dynamics", lw=0.3)
plot!(solutions[1].t,[solutions[1].u[i][2] for i in 1:length(solutions[1].t)], labels="Dead Cells", color="red", lw=0.2)
for j in 2:nsins
    p=plot!(solutions[j].t, [solutions[j].u[i][1] for i in 1:length(solutions[j].t)], labels="", color= "blue", lw=0.2)
    q=plot!(solutions[j].t,[solutions[j].u[i][2] for i in 1:length(solutions[j].t)], labels="", color="red", lw=0.2)
    display(p)
    display(q)
end
oprobbd = ODEProblem(bd_model, u0, tspan, p)
osolbd = solve(oprobbd, reltol=1e-10,abstol=1e-10)
plot!(osolbd.t, [osolbd.u[i][1] for i in 1:length(osolbd.t)], labels="Deterministic Solution", color="black", lw=3)
plot!(osolbd.t, [osolbd.u[i][2] for i in 1:length(osolbd.t)], labels="", color="black", legend=:left, lw=3, title="Birth-Death Model Showing Stochastic Behavior")

################################################################################################################################

# What is the difference between the birth-death model and the exponential model?

using DiffEqBiological
using DifferentialEquations
using Plots
using Colors
#using ParameterizedFunctions

mtdna_model = @reaction_network mtdna begin
  b, wildtype --> 2wildtype
  d, wildtype  --> null
  bmut, mut --> 2mut
  dmut, mut --> null
  m, wildtype --> wildtype + mut
  # can we have wildtype --> 2mut
end b d bmut dmut m

# parameter values

b = 0.001875*365
d = 0.001875*365
bmut = 0.001875*365
dmut = 0.001875*365
m = (1e-5)*365

# initial conditions

wildtype0 = 100.0
mut0 = 0.0
null0 = 0.0
dummy0 = 1.0
tspan= (0.0, 100.0)
p = (b,d,bmut,dmut,m)
u0 = [wildtype0, mut0, null0]

sim_to_tot = function(res, totmax, b_small, b_big, replicative_advantage)
  u0 = res.u[end]
  tot_mtDNA0 = u0[1] + u0[3]
  pnew = collect(p)
  if tot_mtDNA0 >= totmax
    pnew[1] = b_small
    pnew[3] = b_small * replicative_advantage
  else
    pnew[1] = b_big
    pnew[3] = b_big * replicative_advantage
  end
  pnewer = (pnew...,)
  oprobmtdna = ODEProblem(mtdna_model, u0, tspan, pnewer)#, callback=cset)
  osolmtdna = solve(oprobmtdna)
  totmtDNA = [osolmtdna.u[i][1] + osolmtdna.u[i][3] for i in 1:length(osolmtdna.t)];
  triggered = [t>=totmax for t in totmtDNA];
  lastres = findfirst(triggered)-1;
  times = [t for t in osolmtdna.t[1:lastres]]
  species = [u for u in osolmtdna.u[1:lastres]]
  res = (t = times, u = species)
  return(res)
end

res = (t = [0.0], u = [u0])
current_time = res.t[1]
allres = []
while current_time <= tspan[2]
  global current_time, allres, b
  resnew = sim_to_tot(res, 105.0, 0.0, b, 1.0)
  ttotal = [t + current_time for t in resnew.t]
  current_time = ttotal[end]
  updatedres = (t=ttotal, u=resnew.u)
  push!(allres,updatedres)
end

times = collect(Iterators.flatten([res.t for res in allres]))
species = collect(Iterators.flatten([res.u for res in allres]))
osolmtdna = (t=times,u=species)

plot(osolmtdna.t, [osolmtdna.u[i][1] for i in 1:length(osolmtdna.t)], labels="Wildtype", color= "blue")
plot!(osolmtdna.t, [osolmtdna.u[i][3] for i in 1:length(osolmtdna.t)], labels="Mutant", color="red")
plot!(osolmtdna.t, [osolmtdna.u[i][1]+osolmtdna.u[i][3] for i in 1:length(osolmtdna.t)], labels="Total", color="black")

using DiffEqBiological
using DifferentialEquations
using Plots
using Colors

mtdna_model = @reaction_network mtdna begin
  b, wildtype --> 2wildtype
  d, wildtype  --> null
  bmut, mut --> 2mut
  dmut, mut --> null
  m, wildtype --> wildtype + mut
  # can we have wildtype --> 2mut
end b d bmut dmut m

# parameter values

b = 0.001875*365
d = 0.001875*365
bmut = 0.001875*365
dmut = 0.001875*365
m = (1e-5)*365

# initial conditions

wildtype0 = 100.0
mut0 = 10.0
null0 = 0.0
dummy0 = 1.0
tspan= (0.0, 100.0)
p = (b,d,bmut,dmut,m)
u0 = [wildtype0, mut0, null0]

sim_to_tot2 = function(res, totmax, b_small, b_big, replicative_advantage)
  u0 = res.u[end]
  tot_mtDNA0 = u0[1] + u0[3]
  pnew = collect(p)
  if tot_mtDNA0 >= totmax
    pnew[1] = b_small
    pnew[3] = b_small * replicative_advantage
  else
    pnew[1] = b_big
    pnew[3] = b_big * replicative_advantage
  end
  pnewer = (pnew...,)
  probmtdna = DiscreteProblem(u0, tspan ,pnewer)
  jump_prob = JumpProblem(probmtdna,Direct(),mtdna_model)
  solmtdna = solve(jump_prob,FunctionMap())
  totmtDNA = [solmtdna.u[i][1] + solmtdna.u[i][3] for i in 1:length(solmtdna.t)];
  triggered = [t>=totmax for t in totmtDNA];
  lastres = findfirst(triggered)-1;
  times = [t for t in solmtdna.t[1:lastres]]
  species = [u for u in solmtdna.u[1:lastres]]
  res = (t = times, u = species)
  return(res)
end

res = (t = [0.0], u = [u0])
current_time = res.t[1]
allres = []
while current_time <= tspan[2]
  global current_time, allres, b
  resnew = sim_to_tot2(res, 105.0, 0.0, b, 1.0)
  ttotal = [t + current_time for t in resnew.t]
  current_time = ttotal[end]
  updatedres = (t=ttotal, u=resnew.u)
  push!(allres,updatedres)
end

times = collect(Iterators.flatten([res.t for res in allres]))
species = collect(Iterators.flatten([res.u for res in allres]))
solmtdna = (t=times,u=species)

plot(solmtdna.t, [solmtdna.u[i][1] for i in 1:length(solmtdna.t)], labels="Wildtype", color= "blue")
plot!(solmtdna.t, [solmtdna.u[i][3] for i in 1:length(solmtdna.t)], labels="Mutant", color="red")
plot!(solmtdna.t, [solmtdna.u[i][1] + solmtdna.u[i][3] for i in 1:length(solmtdna.t)], labels="Total", color="black", legend=:outerright, title="mtDNA Population Dynamics Within a Cell", xaxis="Time (Years)", yaxis="Population Size")

nsins=10
probmtdna = DiscreteProblem(u0, tspan ,p)
jump_prob = JumpProblem(probmtdna,Direct(),mtdna_model)
solutions =[solve(jump_prob,FunctionMap()) for i in 1:nsins]
plot(solutions[1].t, [solutions[1].u[i][1] for i in 1:length(solutions[1].t)], labels="Wildtype", color= "blue", xaxis="Time", yaxis="Population Size", title="mtDNA Populations Dynamics", lw=0.3)
plot!(solutions[1].t,[solutions[1].u[i][2] for i in 1:length(solutions[2].t)], labels="Mutant", color="red", lw=0.3)
for j in 2:nsins
    p=plot!(solutions[j].t, [solutions[j].u[i][1] for i in 1:length(solutions[j].t)], labels="", color= "blue", lw=0.3)
    q=plot!(solutions[j].t,[solutions[j].u[i][3] for i in 1:length(solutions[j].t)], labels="", color="red", lw=0.3)
    display(p)
    display(q)
end
oprobmtdna = ODEProblem(mtdna_model, u0, tspan, p)
osolmtdna = solve(oprobmtdna)
plot!(osolmtdna.t, [osolmtdna.u[i][1] for i in 1:length(osolmtdna.t)], labels="Deterministic Continuous Solution", color="black", lw=2)
plot!(osolmtdna.t, [osolmtdna.u[i][3] for i in 1:length(osolmtdna.t)], labels="", color="black", legend=:right, lw=2)

# What happens if a cells is born with no mutations?

# What happens if a cell inherits a lot of mutations?  70%?

# What happens in both of the scenarios above if bmut > b?


"""
    combination(n, r)

returns the number of ways picking r unordered outcomes from
n possibilities, without repetition

# Arguments:
- `n`: Positive integers of items to choose from
- 'r': Positive integers of items to choose

Contributed By:- [Mayur Dahibhate](https://github.com/mayurdahibhate)
"""

function combination(n, r)

    function factorial(n)
        fact = 1

        if n == 0 || n == 1
            return fact
        end

        for i = 1:n
            fact = fact * i
        end

        return fact
    end

    comb = factorial(n) / (factorial(r) * factorial(n - r))

    return convert(Int64, comb)
end


"""
Julia program to find the Delannoy number for a given rectangular grid.
A Delannoy number describes the number of paths from the southwest corner
(0, 0) of a rectangular grid to the northeast corner (m, n) using only single steps north, northeast, or east.
Delannoy number is named after the French mathematican Henri Delannoy.
"""

function delannoy_number(r, c)

    # Initialize the dp array with '0' as value.
    dp = zeros(Int, c + 1, r + 1)

    for i in 1:c
        dp[1,i] = 1
    end
    for i in 2:c+1
        dp[i,1] = 1
    end
    # From Each point calculate the number of paths, to the north-east point, that can be reached by
    # traversing through the immediate right or immediate top or immediate top-right point.
    for i in 2:c+1
        for j in 2:r+1
            dp[i,j] = dp[i-1,j] + dp[i-1,j-1] + dp[i,j-1]
        end
    end
    return dp[c+1,r+1]
end

print("Enter the co-ordinates of the north-east corner.\nEnter the row co-ordinate: ")
x = readline()
x = parse(Int, x)
print("Enter the column co-ordinate: ")
y = readline()
y = parse(Int, y)
res = delannoy_number(x, y)
print("The Delannoy number of the given grid is $res.")

"""
Time Complexity - O(x * y), where `x`, `y` is the given co-ordinates.
Space Complexity - O(1)

SAMPLE INPUT AND OUTPUT

SAMPLE I

Enter the co-ordinates of the north-east corner.
Enter the row co-ordinate: 5
Enter the column co-ordinate: 6
The Delannoy number of the given grid is 3653.
"""


@doc raw"""
    triag_rng(a, b, m, shape=1; seed=nothing)

Generate a `shape` element array of random variables from a Triangular(`a`, `b`, `m`) distribution. Optionally you can set a specific seed.

# Notes

The Triangular distribution is given by:

```math
f(x, a, b, m) = \begin{cases}
\frac{2(x-a)}{(m-a)(b-a)}  & \text{if  } a < x ≤ m, \\
\frac{2(b-x)}{(b-m)(b-a)}  & \text{if  } m < x ≤ c, \\
0 & \text{otherwise}
\end{cases}
```

# Examples

```julia-repl
julia> triag_rng()
1-element Vector{Float64}:
 0.3559088458688944

julia> triag_rng(0,7,2,5)
5-element Vector{Float64}:
 1.3758072115332673
 6.049452463477447
 6.042781317411027
 2.914959243260448
 4.454707036522528
```

# References

W. Stein and M. Keblis. A new method to simulate the triangular distribution. Mathematical and Computer Modelling, Volume 49, Issues 5–6, 2009.
"""
function triag_rng(a=0,b=1,m=.5,shape=1)
    u = get_std_uniform(shape)
    v = get_std_uniform(shape)
    t_min = min.(u,v)
    t_max = max.(u,v)
    c = (m-a)/(b-a)
    t = ((1-c) .* t_min) .+ (c .* t_max)
    X = a .+ (b-a) .*t
    return X
end


using Random, Distributions, LinearAlgebra
using Plots; pyplot()

function sum_of_squares(v)
    """Computes the sum of squared elements in v"""
    return dot(v, v)
end

function difference_quotient(f, x, h)
    return (f(x + h) - f(x)) / h
end

function square(x)
    return x * x
end

function derivative(x)
    return 2 * x
end

@assert 4 <= difference_quotient(square, 2, .001) <=4.01

function squared_distance(v, w)
    """Computes (v_1 - w_1) ** 2 + ... + (v_n - w_n) ** 2"""
    return sum_of_squares((v - w))
end

function distance(v, w)
    """Computes the distance between v and w"""
    return sqrt(squared_distance(v, w))
end

function partial_difference_quotient(f, v, i, h)
        """Returns the i-th partial difference quotient of f at v"""
        # w = [v_j + (h if j == i else 0) for j, v_j in enumerate(v)]
        w = copy(v)
        for j in 1:length(v)
            v_j = v[j]
            if(j==i)
                w[j] = v_j + h
            else
                w[j] = v_j
            end
        end

        return (f(w) - f(v)) / h
end

function add(v, w)
    """Adds corresponding elements"""
    @assert length(v) == length(w)
    return v+w
end

function scalar_multiply(c, v)
    """Multiplies every element by c"""
    return [c * v_i for v_i in v]
end

function gradient_step(v, gradient, step_size)
    """Moves `step_size` in the `gradient` direction from `v`"""
    @assert length(v) == length(gradient)
    step = scalar_multiply(step_size, gradient)
    return add(v, step)
end

function sum_of_squares_gradient(v)
    return [2 * v_i for v_i in v]
end

function custom_fun(v)
    sign = -1
    if(v[2] > 4)
        sign = 1
    end
    return v[1]*v[1] + sign * v[2]*v[2] + 16
end

function estimate_gradient(f, v, h = 0.0001)
    return [partial_difference_quotient(f, v, i, h) for i in 1:length(v)]
end
# let v = [rand(-10: 10) for i in 0:2]
let v = [2.0, 2.0]
    for epoch in 1:1000
        # grad = sum_of_squares_gradient(v)    # compute the gradient at v
        grad = estimate_gradient( custom_fun,v)
        v = gradient_step(v, grad, -0.001)    # take a negative gradient step
    end

    # @assert distance(v, [0, 0]) < 0.001    # v should be close to 0
    @assert custom_fun(v) < .1
end

xs = -10: 11
actuals = [derivative(x) for x in xs]
estimates = [difference_quotient(square, x, 0.001) for x in xs]

# plot to show they're basically the same
plot(title = "Actual Derivatives vs. Estimates")
plot!(xs, actuals, label="Actual", seriestype = :scatter, color = "red", markersize = 7)       # red  x
plot!(xs, estimates, label="Estimate", seriestype = :scatter, color = "blue", markershape= :x)   # blue +
# plot(legend=loc=9)
# plot.show()

result = estimate_gradient(sum_of_squares,[2.0 2.0])
println(result) #should be equal to [4.0 4.0]

function error(v)
    x, y, slope, intercept = v
    predicted = slope * x + intercept
    error = (predicted - y)^2
    return error
end

println(error([2, 1 , 3, 5]))

ex_vector = [2.0, 1.0 , 2.0, 5.0]
println(partial_difference_quotient(error,ex_vector,3,.001))

function linear_gradient(x, y, theta)
    vector = [x,y, theta...]
    error_diff_estimate = [partial_difference_quotient(error,vector,i,.001)
                        for i in 3:4]

end
error_diff_estimate = linear_gradient(2.0,1.0, [2.0, 5.0])
println(error_diff_estimate)

inputs = [(x, 20 * x + 5) for x in -50: 49]

function vector_mean(vectors)
    """Computes the element-wise average"""
    n = length(vectors)
    return scalar_multiply(1/n, sum(vectors))
end

ud = Uniform(-1,1)
let theta = [rand(ud), rand(ud)]

learning_rate = 0.001
for epoch in 1:3000
    grad = vector_mean([linear_gradient(x,y, theta) for (x,y) in inputs])

    theta = gradient_step(theta, grad, -learning_rate)
    println(epoch, theta)
end

slope, intercept = theta
@assert 19.9 < theta[1] < 20.1
@assert 4.9 < intercept < 5.1
end

#finished


exp(π) - π

# Solve x^2 + 5x + 6 = 0 using the quadratic formula (real arithmetics)

# Coefficients a,b,c in ax^2 + bx + c = 0
a = 1
b = 5
c = 6

# The quadratic formula
d = sqrt(b^2 - 4*a*c)
r1 = (-b - d) / 2a
r2 = (-b + d) / 2a
println("The roots are ", r1, " and ", r2)

function myfunc(x,y)
    x + y
end

myfunc(1,2)

myfunc2(x,y) = x + 2y
myfunc2(3,5)

myfunc3 = (x,y) -> x + 3y
myfunc3(3,5)

# Same thing without giving the function a name:
((x,y) -> x + 3y)(3,5)

function mynewfunc(x,y)
    out1 = x + y
    out2 = out1 * (2x + y)
    out1, out2
end

y1, y2 = mynewfunc(2,1)

function real_roots_of_quadratic(a,b,c)
    # Compute the real roots of the quadratic ax^2 + bx + c = 0
    d = sqrt(b^2 - 4*a*c)
    r1 = (-b - d) / 2a
    r2 = (-b + d) / 2a
    r1, r2
end

real_roots_of_quadratic(1, 5, 6)

real_roots_of_quadratic(-1, 5, 6)


"""
    beta_rng(α, β, shape=1; seed=nothing)

Generate a `shape` element array of random variables from a Beta(`α`, `β`) distribution. Optionally you can set a specific seed.

# Notes

The Beta distribution is given:

``f(x,α,β) = \\frac{x^{α-1 (1-x)^{β-1}}}{Β(α,β)} \\quad x \\in [0,1]``

where ``Β(α,β) = \\frac{Γ(α)Γ(β)}{Γ(α+β)}``

# Examples

```julia-repl
julia> beta_rng(1,2)
1-element Vector{Float64}:
 0.44456674672905633

julia> beta_rng(1, 2, (2,2))
2×2 Matrix{Float64}:
 0.0792132  0.595657
 0.737615   0.649721

```

# References

D.P. Kroese, T. Taimre, Z.I. Botev. Handbook of Monte Carlo Methods.
  Wiley Series in Probability and Statistics, John Wiley & Sons, New York, 2011.

Law, A. Simulation modeling and analysis, 5th Ed. McGraw Hill Education, Tuscon, 2013.
"""
function beta_rng(α::Real, β::Real, shape::Union{Int, Tuple{Vararg{Int}}}=1; seed::Union{Int, Nothing}=nothing)
    seed_setter(seed)
    Y₁ = gamma_rng(α, 1, shape, seed=seed)
    Y₂ = gamma_rng(β, 1, shape, seed=seed)
    X = Y₁ ./ (Y₁ .+ Y₂)
    return X
end


#= Finding the Greatest Common Divisor of 2 numbers using the Euclidean
Formula to lessen the time complexity.
=#

## Function

function GCD(a, b)
    if (a == 0)
        return b
    end
    if (b == 0)
        return a
    end
    if (a < b)
        a, b = b, a
    end
    ans = a % b
    while (ans != 0)
        a = b
        b = ans
        ans = a % b
    end
    return b
end

## Input

a = readline()
a = parse(Int64, a)
b = readline()
b = parse(Int64, b)

## Calling the function

GCD(a, b)

#=
Sample Test case:
Input:
    a = 1001  b = 2352
Output:
    7
Time complexity: O( log (min(a,b)) )
=#


# # Huber regression

# This example can be found here: <https://web.stanford.edu/~boyd/papers/pdf/cvx_applications.pdf>.
# Here we set `big_example = false` to only generate a small example which takes less time to run.
big_example = false
if big_example
    n = 300
    number_tests = 50
else
    n = 100
    number_tests = 20
end

# Generate data for Huber regression.
using Random
Random.seed!(1);
number_samples = round(Int,1.5*n);
beta_true = 5*randn(n);
X = randn(n, number_samples);
Y = zeros(number_samples);
v = randn(number_samples);

#-

## Generate data for different values of p.
## Solve the resulting problems.
using Convex, SCS, Distributions
lsq_data = zeros(number_tests);
huber_data = zeros(number_tests);
prescient_data = zeros(number_tests);
p_vals = range(0, stop=0.15, length=number_tests);
for i=1:length(p_vals)
    p = p_vals[i];
    ## Generate the sign changes.
    factor = 2 * rand(Binomial(1, 1-p), number_samples) .- 1;
    Y = factor .* X' * beta_true + v;

    ## Form and solve a standard regression problem.
    beta = Variable(n);
    fit = norm(beta - beta_true) / norm(beta_true);
    cost = norm(X' * beta - Y);
    prob = minimize(cost);
    solve!(prob, SCS.Optimizer(verbose=0));
    lsq_data[i] = evaluate(fit);

    ## Form and solve a prescient regression problem,
    ## i.e., where the sign changes are known.
    cost = norm(factor .* (X'*beta) - Y);
    solve!(minimize(cost), SCS.Optimizer(verbose=0))
    prescient_data[i] = evaluate(fit);

    ## Form and solve the Huber regression problem.
    cost = sum(huber(X' * beta - Y, 1));
    solve!(minimize(cost), SCS.Optimizer(verbose=0))
    huber_data[i] = evaluate(fit);
end

#-
using Plots

plot(p_vals, huber_data, label="Huber", xlabel="p", ylabel="Fit" )
plot!(p_vals, lsq_data, label="Least squares")
plot!(p_vals, prescient_data, label="Prescient")
#-

## Plot the relative reconstruction error for Huber and prescient regression,
## zooming in on smaller values of p.
indices = findall(p_vals .<= 0.08);
plot(p_vals[indices], huber_data[indices], label="Huber")
plot!(p_vals[indices], prescient_data[indices], label="Prescient")


#=
							Even Fibonacci numbers

	Each  new term in the Fibonacci sequence is generated by adding the previous
	two terms. By starting with 1 and 2, the first 10 terms will be:

					1, 2, 3, 5, 8, 13, 21, 34, 55, 89, ...

	By  considering  the  terms  in  the  Fibonacci sequence whose values do not
	exceed four million, find the sum of the even-valued terms.

								Answer: 4613732
=#

function fibonacci(limit::Int)
    array = [1, 2]
    flag = true
    i = 2

    while flag
        new_value = array[i]+array[i-1]

        if new_value < limit
            push!(array, new_value)
            i += 1
        else
            flag = false
        end
    end

    return array
end

println(sum(filter(iseven, fibonacci(4_000_000))))


#=
(a, ya) (b, yb) (c, yc)
====
q(x) = p1 + p2*x + p3 * x^2
====
ya = p1 + p2*a + p3 * a^2
yb = p1 + p2*b + p3 * b^2
yc = p1 + p2*c + p3 * c^2
=====
lagrange’s interpolating polynomial:
q(x) = ya * ((x-b)(x-c))/((a-b)(a-c)) +
       yb * ((x-a)(x-c))/((b-a)(b-c)) +
       yc * ((x-a)(x-b))/((c-a)(c-b))
=====
q'(x) = 1/2 * (ya(b^2-c^2)+yb(c^2-a^2)+yc(a^2-b^2))/(ya(b-c) + yb(c-a) + yc(a-b))
=#

function quadratic_fit_search(f, a, b, c, n)
    ya, yb, yc = f(x), f(x), f(c);
    for i in 1:n-3
        x = 0.5 * (ya(b^2-c^2)+yb(c^2-a^2)+yc(a^2-b^2))/
                  (ya(b-c) + yb(c-a) + yc(a-b));
        yx = f(x);
        if x > b
            if yx > yb
                c, yc = x, yx;
            else
                a, ya, b, yb = b, yb, x, yx;
            end
        elseif x < b
            if yx > yb
                a, ya = x, yx;
            else
                c, yc, b, yb = b, yb, x, yx;
            end
        end
    end
    return (a, b, c);
end


using LinearAlgebra
using SparseArrays
using Plots
using ForwardDiff

"This routine runs a convergence test for the heat equation using either Backwards
Euler or Crank-Nicolson in time and finite differences in space. The exact solution
is generated using the method of manufactured solutions."

uexact(x,t) = log(2+sin(pi*x))*exp(-t)*t
dudx_exact(x,t) = ForwardDiff.derivative(x->uexact(x,t),x)
# du/dt - d^2u/dx^2 = f
f(x,t) = ForwardDiff.derivative(t->uexact(x,t),t) -
         ForwardDiff.derivative(x->dudx_exact(x,t),x)
u0(x) = uexact(x,0)
α(t) = uexact(-1,t)
β(t) = uexact(1,t)

# m = number of grid points
function build_FD(m,f,α,β)
    x = LinRange(-1,1,m+2)
    xint = x[2:end-1]
    h = x[2]-x[1]
    A = (1/h^2) * spdiagm(0=>2*ones(m),-1=>-ones(m-1),1=>-ones(m-1))

    function F(t)
        b = f.(xint,t)
        b[1] += α(t)/h^2
        b[m] += β(t)/h^2
        return b
    end
    return A,x,xint,h,F
end

function solve_backward_Euler(m, f, α, β, u0, T, dt)

    A,x,xint,h,F = build_FD(m,f,α,β)

    u = u0.(xint)
    Nsteps = ceil(Int,T/dt)
    dt = T/Nsteps
    for i = 1:Nsteps
        t = i*dt
        u .= (I + dt*A)\(u .+ dt*F(t))
    end
    return u,x,xint,h
end

function solve_Crank_Nicolson(m, f, α, β, u0, T, dt)

    A,x,xint,h,F = build_FD(m,f,α,β)

    u = u0.(xint)
    Nsteps = ceil(Int,T/dt)
    dt = T/Nsteps
    for i = 1:Nsteps
        tprev = (i-1)*dt
        t = i*dt
        u .= (I + .5*dt*A)\((I - .5*dt*A)*u + .5*dt*(F(t)+F(tprev)))
    end
    return u,x,xint,h
end

mvec = 2 .^ (2:8)
hvec = zeros(length(mvec))
hvec = Float64[]
err1 = Float64[]
err2 = Float64[]
for (i,m) in enumerate(mvec)

    T = 1.0
    dt = 5/m # proportional to h

    u,x,xint,h = solve_backward_Euler(m,f,α,β,u0,T,dt)
    push!(err1,maximum(@. abs(uexact(xint,T) - u)))

    u,x,xint,h = solve_Crank_Nicolson(m,f,α,β,u0,T,dt)
    push!(err2,maximum(@. abs(uexact(xint,T) - u)))

    push!(hvec,h)
end
plot(hvec,err1,marker=:dot,label="Max error (backwards Euler)")
plot!(hvec,err2,marker=:dot,label="Max error (Crank-Nicolson)")
plot!(hvec,hvec,linestyle=:dash,label="O(h)")
plot!(hvec,hvec.^2,linestyle=:dash,label="O(h^2)")
plot!(xaxis=:log,yaxis=:log,legend=:bottomright)


# Input data for generation of the cellular complex shown in figure of Data 2.1.2

V = [
    0.0 1.5 3.0 1.0 1.5 2.0 1.0 1.5 2.0 0.0 1.5 3.0;
    0.0 0.0 0.0 1.0 1.0 1.0 2.0 2.0 2.0 3.0 3.0 3.0
];
EV = [
    [1,  2], [2,  3], [ 4,  5], [ 5,  6],
    [7,  8], [8,  9], [10, 11], [11, 12],
    [1, 10], [4,  7], [ 6,  9], [ 3, 12],
    [2,  5], [8, 11]
];
FV = [
    [1, 2, 4, 5, 7, 8, 10, 11],
    [2, 3, 5, 6, 8, 9, 11, 12],
    [4, 5, 6, 7, 8, 9]
];

# The function K returns the Julia SparseArrays matrix providing
#  the characteristic matrix K𝑟 given as input an array CV
#  specifying each 𝑟-cell as array of vertex indices.

function K( CV )
    I = vcat( [ [k for h in CV[k]] for k=1:length(CV) ]...)
    J = vcat( CV...)
    X = Int8[1 for k=1:length(I)]
    return SparseArrays.sparse(I, J, X)
end

# Generation of the sparse binary matrices

copEV = K(EV);
copFV = K(FV);

Matrix(copEV)
Matrix(copFV)

if todisplay
    VV = [[k] for k in 1:size(V,2)];
    GL.VIEW( GL.numbering(1.0)( (V, [VV, EV]),GL.COLORS[1],0.1 ) );
end;

# Boundary matrix generation
# we construct the sparse matrix [𝜕2] from the product of characteristic matrices EV and the transposed FV

Matrix( copEV * copFV' )

copEF = (copEV * copFV') .÷ 2;

Matrix(copEF)

b1 = (copEF * sparse([1 1 1])') .% 2
b2 = (copEF * sparse([1 1 0])') .% 2

f1 = SparseArrays.findnz(b1)[1]
f2 = SparseArrays.findnz(b2)[1]

e1 = EV[f1]
e2 = EV[f2]

if todisplay
    GL.VIEW( GL.numbering(1.0)( (V, [VV, e1]),GL.COLORS[1],0.1 ) );
    GL.VIEW( GL.numbering(1.0)( (V, [VV, e2]),GL.COLORS[1],0.1 ) );
end;


#=
In this example we'll compare the optimization of a relatively simple problem: Logistic Regression.

Consider a binary classification dataset
  $\Omega = \{(x_i, y_i) \subset \mathbb{R}^p\times\{0,1\}:\ i = 1, \dots, n\}$.

The logistic regression problem defines a model
$h_{\beta}(x) = \sigma(\hat{x}^T \beta) = \sigma(\beta_0 + x^T\beta_{1:p})$,
where
$\hat{x} = \begin{bmatrix} 1 \\ x \end{bmatrix}$.
The value of $\beta$ is found by finding the maximum of the log-likelihood function

$$\ell(\beta) = \frac{1}{n} \sum_{i=1}^n y_i \ln \big(h_{\beta}(x_i)\big) + (1 - y_i) \ln\big(1 - h_{\beta}(x_i)\big).$$

We'll input the gradient of this function manually. It is given by

$$\nabla \ell(\beta) = \frac{1}{n} \sum_{i=1}^n \big(y_i - h_{\beta}(x_i)\big) \hat{x}_i = \frac{1}{n} \begin{bmatrix} e^T \\ X^T \end{bmatrix} (y - h_{\beta}(X)), $$

where $e$ is the vector with all components equal to 1.

Download [data.csv](https://github.com/abelsiqueira/julia-tutorial-at-nlesc/raw/main/assets/data.csv) to follow the example.

In Python, we can define the function $-\ell(\beta)$ to minimize, and its derivatives, as follows:

```python
import numpy as np
import pandas as pd
from scipy.optimize import minimize

df = pd.read_csv('data.csv')
X = df.iloc[:,0:-1].values
y = df.iloc[:,-1].values
n, p = X.shape

def sigmoid(t):
    return 1 / (1 + np.exp(-t))

def myfun(beta):
    hbeta = sigmoid(beta[0] + X.dot(beta[1:]))
    return -np.sum(
        y * np.log(hbeta + 1e-8) + (1 - y) * np.log(1 - hbeta + 1e-8)
    ) / n

def myjac(beta):
    hbeta = sigmoid(beta[0] + X.dot(beta[1:]))
    dif = hbeta - y
    return np.concatenate((np.array([np.sum(dif)]), X.T.dot(dif))) / n

beta0 = np.zeros(p + 1)
res = minimize(myfun, beta0, jac=myjac, method='l-bfgs-b', tol=1e-4)
beta = res.x
hbeta = sigmoid(beta[0] + X.dot(beta[1:]))
ypred = np.round(hbeta)
accuracy = sum(y == ypred) / n
print('accuracy = ', accuracy)
```
If you're using Jupyter, I also ask you to run
```
%%timeit
res = minimize(myfun, beta0, jac=myjac, method='l-bfgs-b', tol=1e-4)
```
So that we can compare the time

Here's a similar Julia version
=#
using CSV, DataFrames, Optim, LinearAlgebra

path = joinpath("assets", "data.csv")
df = DataFrame(CSV.File(path))
X = Matrix(df[:,1:end-1])
y = df[:,end]
n, p = size(X)

sigmoid(t) = 1 / (1 + exp(-t))

function myfun(β, X, y)
  @views hβ = sigmoid.(β[1] .+ X * β[2:end])
  out = sum(
    yᵢ * log(ŷᵢ + 1e-8) + (1 - yᵢ) * log(1 - ŷᵢ + 1e-8)
    for (yᵢ, ŷᵢ) in zip(y, hβ)
  )
  return -out / length(y)
end

function myjac(out, β, X, y)
  n = length(y)
  @views δ = (sigmoid.(β[1] .+ X * β[2:end]) - y) / n
  out[1] = sum(δ)
  out[2:end] .= X' * δ
  return out
end

output = optimize(
  β -> myfun(β, X, y),
  (out, β) -> myjac(out, β, X, y),
  zeros(p + 1),
  LBFGS(),
  Optim.Options(
    g_tol = 1e-4,
  ),
)

β = Optim.minimizer(output)
hβ = sigmoid.(β[1] .+ X * β[2:end])
ŷ = round.(hβ)
accuracy = sum(y .== ŷ) / n

#=
The beginning of the codes is similar: read the file, create `X` and `y`.

We then define the functions. Some main differences:
- Instead of writing the objective vectorized, I use a loop over the indices.
- To avoid creating a vector for `β[2:end]`, I use `@views` at the beginning of the line. The macro automatically transforms `β[2:end]` into `view(β, 2:end)`.
- The Julia `myjac` function receives an additional argument for the memory that it should reuse (`gx`) for the output.

Now, the specific part. In Python, we use SciPy's optimize, and in Julia we're using Optim.jl.
The arguments are similar, although Optim's arguments are positional.

We can benchmark Optim's code to compare against SciPy's optimize:
=#
using BenchmarkTools

β₀ = zeros(p + 1)
@benchmark optimize(
  β -> myfun(β, X, y),
  (out, β) -> myjac(out, β, X, y),
  β₀,
  LBFGS(),
  Optim.Options(
    g_tol = 1e-4,
  ),
)
#=
In Julia, we don't have a single optimization library.
In fact, I'm co-creator of the [JuliaSmoothOptimizers](https://juliasmoothoptimizers.github.io) organization, which "competes" against Optim.
We focus on tools for Nonlinear Optimization developers, instead of focusing on users, so our basic interface is a lot more raw.
On the other hand, we claim that our methods are faster, but notice my bias.

NLPModels alone doesn't have a simple user interface, so we create a struct `MyLogisticRegression` to hold the problem and to indicate the objective and gradient.
The struct fields are not obvious and you have to follow the docs of NLPModels to learn how to do it.
But you can check the speed gain.
=#

using JSOSolvers, Logging, NLPModels

struct MyLogisticRegression <: AbstractNLPModel{Float64, Vector{Float64}}
  meta :: NLPModelMeta{Float64, Vector{Float64}}
  counters :: Counters
  X
  y

  function MyLogisticRegression(X, y)
    meta = NLPModelMeta(p+1, x0=zeros(p+1))
    return new(meta, Counters(), X, y)
  end
end

NLPModels.obj(nlp :: MyLogisticRegression, x) = myfun(x, nlp.X, nlp.y)
NLPModels.grad!(nlp :: MyLogisticRegression, x, gx) = myjac(gx, x, nlp.X, nlp.y)

nlp = MyLogisticRegression(X, y)
output = lbfgs(nlp, atol=1e-4, rtol=1e-4)
println(output)

@benchmark with_logger(NullLogger()) do
  lbfgs(nlp, atol=1e-4, rtol=1e-4)
end

# TODO: Check progress of ManualNLPModels (check after 18-Oct)

#=
One noteworthy aspect of this comparison is that Python uses the L-BFGS-B Fortran algorithm, which is a reference implementation of a classic method.
The Julia versions are pure Julia.
=#


#= udemy.com code challenge
from Prof. Mike X Cohen's course in Linear Algebra
https://www.udemy.com/course/linear-algebra-theory-and-implementation

# Is the dot product commutative?
# A*B' == B*A' ?
=#

using LinearAlgebra

# generate two 100-element random row vectors, transposed from default column layout.
a = randn(100)'
b = randn(100)'

# display vectors (in truncated form)
println("Vector a with 100 elements:")
display(a)
println("\n")

println("Vector b of the same size:")
display(b)
println("\n")

# calculate dot products
dp1 = dot(a, b) # a * b' also works
dp2 = dot(b, a) # b * a'

# display them
println("Dot product of a and b is $(dp1)")
println("Dot product of b and a is $(dp2)\n")

# are they equal?
if dp1 == dp2
  println("The dot product of the 100-element vector is commutative.\n")
else
  println("Dot product is not commutative.\n")
end

# generate 1 row, 2 column array with random integers from -10:10
c = rand(-10:10, 1, 2)

d = rand(-10:10, 1, 2)

println("Array c = $c")
println("Array d = $d\n")

dp3 = c * d'
dp4 = d * c'

println("Dot product of c and d is $(dp3)")
println("Dot product of d and c is $(dp4)")

if dp3 == dp4
  println("Thus, the dot product of the 2-element vector is commutative.")
else
  println("Dot product is not commutative")
end


using LinearAlgebra
using Dates
using Plots

include("qr_factorization.jl")
include("truncated_svd.jl")

"""
verifies <num_tests> times that ||Q*R - A||_2 <= min_error
"""
function test_qr_A(num_tests, min_error= 1e-13)
    errors = 0

    for i = 1:num_tests

        #generating random matrix
        rows = rand(10:100)
        columns = rand(10:rows)
        r = rand(10:columns)
        #using SVD to obtain a rank r matrix
        A = approx_matrix_svd(rand(rows,columns),r)

        #calculating QR factorization
        QR, v, u = allocate_matrices(A)
        Q = rand(rows, columns)
        qr_factorization!(A,QR,v,u)
        get_Q!(QR,Q)
        R = triu(QR)[1:columns,:]

        #checking ||QR - A||
        if (norm(Q*R - A) > min_error)
            errors += 1
        end

    end
    return errors
end

"""
verifies <num_tests> times that ||R - R_j||_2 <= min_error where R is the R
computed by our QR factorization while R_j is the R computed by the
JULIA's implementation
"""
function test_qr_R(num_tests, min_error = 1e-10)
    errors = 0

    for i = 1:num_tests

        #generating random matrix
        rows = rand(10:100)
        columns = rand(10:rows)
        r = rand(10:columns)
        A = approx_matrix_svd(rand(rows,columns),r)

        #calculating QR factorization
        QR, v, u = allocate_matrices(A)
        qr_factorization!(A,QR,v,u)
        Q_j,R_j = qr(A)
        R = triu(QR)[1:columns,:]

        #checking ||R - R_j||
        if (norm(R - R_j) > min_error)
            errors += 1
        end

    end
    return errors
end

"""
verifies Q_t_times_A() function
"""
function test_qr_multiplication(num_tests, min_error = 1e-10)
    errors = 0

    for i = 1:num_tests
        #generating random matrix
        rows = rand(10:100)
        columns = rand(10:rows)
        r = rand(10:columns)
        A = approx_matrix_svd(rand(rows,columns),r)

        #calculating QR factorization
        QR, v, u = allocate_matrices(A)
        qr_factorization!(A,QR,v,u)
        Q_j,R_j = qr(A)
        W = zeros(rows,columns)
        Q_t_times_A!(QR,A,W)
        W_j = Q_j'*A

        #checking ||W - W_j||
        if (norm(W - W_j) > min_error)
            errors += 1
        end

    end
    return errors
end

function qr_test(A)
    QR, v, u = allocate_matrices(A)
    qr_factorization!(A,QR,v,u)
end

"""
Draws a plot that compares the time (in seconds) required by our implementation and the one built-in in Julia.

The algorithm generates Int((max_dimension - min_dimension) / step) + 1 random matrices of increasing dimension.

At step k, the matrix shape will be (min dimension+((k - 1) * step) x min dimension+((k - 1) * step)). The algorithm terminates when
the matrix shapes reach max dimension.

"""
function test_time_qr(min_dimension, max_dimension; step = 15)
    plotly()
    dim = Int((max_dimension - min_dimension) / step) + 1
    x = zeros(dim)
    y = zeros(dim,2)
    for i = 0:dim-1
        x[i+1] = i * step + min_dimension
        A = rand(Int(x[i+1]),Int(x[i+1]))
        y[i+1,1] = @elapsed qr_test(A)
        y[i+1,2] = @elapsed qr(A)
    end
    plot(x, y, label = ["Our QR" "Julia's QR"], lw = 3)
end


using Pkg
Pkg.activate(pwd())

# # Linear regression
# ## Loading and preparing data

using Plots
using StatsPlots
using RDatasets

iris = dataset("datasets", "iris")
iris[1:5,:]

# ### Exercise:
# We will simplify the goal and estimate the dependence of petal width on petal length. # Create the data $X$ (do not forget to add the bias) and the labels $y$.
#
# Make a graph of the dependence of petal width on petal length.
#
# ---
# ### Solution:

y = iris.PetalWidth
X = hcat(iris.PetalLength, ones(length(y)))

@df iris scatter(
    :PetalLength,
    :PetalWidth;
    label="",
    xlabel = "Petal length",
    ylabel = "Petal width"
)

# ---
#
# ## Training the classifier
# ### Exercise:
# Use the closed-form formula to get the coefficients $w$ for the linear regression. Then
# use the `optim` method derived in the previous lecture to solve the optimization problem
# via gradient descent. The results should be identical.
#
#---
#### Solution:

abstract type Step end

struct GD <: Step
    α::Real
end

optim_step(s::GD, f, g, x) = -s.α*g(x)

function optim(f, g, x, s::Step; max_iter=100)
    for i in 1:max_iter
        x += optim_step(s, f, g, x)
    end
    return x
end

#+

using LinearAlgebra

w = (X'*X) \ (X'*y)
g(w) = X'*(X*w-y)
w2 = optim([], g, zeros(size(X,2)), GD(1e-4); max_iter=10000)

norm(w - w2)

# ---
#
# ### Exercise:
# Write the dependence on the petal width on the petal length. Plot it in the previous
# graph.
#
# ---
# ### Solution:

f_pred(x::Real, w) = w[1]*x + w[2]
x_lims = extrema(iris.PetalLength) .+ [-0.1, 0.1]

@df iris scatter(
    :PetalLength,
    :PetalWidth;
    xlabel = "Petal length",
    ylabel = "Petal width",
    label = "",
    legend = :topleft,
)

plot!(x_lims, x -> f_pred(x,w); label = "Prediction", line = (:black,3))


function bisection(fun, a, b; threshold = 1e-10, maxiter = 100)
    root = (a+b)/2
    iter = 0
    while (abs(fun(root)) > threshold) && (iter < maxiter)
        if fun(a)*fun(root) < 0
            b = root
        else
            a = root
        end
        root = (a+b)/2
        iter += 1
    end
    if iter == maxiter
        println("Maximal number of iterations reached in bisection method.")
    end
    root
end

####################
# Newton iterations
#####################

"The abstract NonlinearSystem type represents a system of non-linear equations in n variables."
abstract type NonlinearSystem
end

# Default numeric type is Float64
# eltype(::Type{S}) where {S <: NonlinearSystem} = Float64

function residual(sys::NonlinearSystem, x)
    result = Array{eltype(sys)}(length(sys))
    residual!(result, sys, x)
end

function jacobian(sys::NonlinearSystem, x)
    J = Array{eltype(sys)}(size(sys))
    jacobian!(J, sys, x)
end

"""
Apply Newton iterations with the given starting values to find the roots of the given
system of equations. A convergence threshold and maximum number of iterations are optional.

The maximal step size in each iteration can be limited using `maxstep`.

Damped Newton iterations can be achieved by supplying damping parameter 0 < alpha < 1.
"""
function newton(sys::NonlinearSystem, x0; threshold = 1e-6, alpha = 1, maxiter = 100, maxstep = 1e5, verbose = false)
    n = length(x0)
    S = Array{eltype(sys)}(undef, n)
    J = Array{eltype(sys)}(undef, n, n)

    residual!(S, sys, x0)
    jacobian!(J, sys, x0)
    c = cond(J)
    threshold = max(threshold, eps(eltype(x0))*c)
    u = J \ S
    if norm(alpha*u) > maxstep
        u = u * maxstep/norm(u)
    end
    x1 = x0 - alpha*u

    iter = 1
    while norm(x1-x0) > threshold && iter < maxiter
        iter += 1
        x0 = x1
        residual!(S, sys, x0)
        jacobian!(J, sys, x0)
        u = J \ S
        if norm(alpha*u) > maxstep
            u = u * maxstep/norm(u)
        end
        x1 = x0 - alpha*u
    end
    if verbose
        if iter == maxiter
            println("Newton: maximum number of iterations, no convergence")
        else
            println("Newton: Converged to solution in ", iter, " iterations.")
        end
    end
    x1,iter,norm(x1-x0)
end

"Invoke Newton but restart with smaller maximal step size if an error is thrown."
function newton_with_restart(args...; maxstep = 1e5, maxiter = 100, verbose = false, options...)
    local nx,iter,normx
    nb_decreased = 0
    try
    nx,iter,normx = newton(args...; maxstep = maxstep, options...)
    catch e
        print("Error thrown by newton: ")
        println(e)
        iter = maxiter
    end
    while iter == maxiter
        verbose && println("Decreasing step size.")
        maxstep /= 2
        try
            nx,iter,normx = newton(args...; maxstep = maxstep, verbose=verbose, options...)
        catch e
            print("Error thrown by newton: ")
            println(e)
            iter = maxiter
        end
        nb_decreased += 1
        if nb_decreased > 20
            error("Too many step decreases in newton_with_restart. Aborting.")
        end
    end
    nx,iter,normx
end


"""
    lu_decompose(mat)
Decomposes a `n x n` non singular matrix into a lower triangular matrix (L) and an upper triangular matrix (U)
"""
function lu_decompose(mat)
	n = mat |> size |> first
	L = zeros(n, n)
	U = zeros(n, n)

	for i in 1:n
		for j in i:n
			s = 0
			for k in 1:i
				s += L[i,k] * U[k,j]
			end
			U[i,j] = mat[i,j] - s
		end

		for k in i:n
			if i == k
				L[i,i] = 1
    			else
				s = 0
				for j in 1:i
					s += L[k,j] * U[j,i]
				end
				L[k,i] = (mat[k,i] - s) / U[i,i]
			end
		end
	end

	return L, U
end


function errorCheck(a1,a2,p)
    if length(a1)!= length(a2)
        println("The arrays have different length")
        return -1
        end
        if p<1
            println("the value attributed to 'p' must be at least 1")
            return -1
        end
    return 0
end

function euclidianDistance(a1,a2)
        if errorCheck(a1,a2,1) !=-1
                return sqrt((sum((a1-a2).^2)))
        end
end

function manhattanDistance(a1,a2)
        if errorCheck(a1,a2,1) !=-1
                return sum(abs.((a1-a2)))
        end
end

function cosineDistance(a1,a2)
        if errorCheck(a1,a2,1) !=-1
                return 1 - sum(a1.*a2)/(sqrt(sum(a1.^2))*sqrt(sum(a2.^2)))
        end
end

function minkowskiDistance(a1,a2,p)
        if errorCheck(a1,a2,p) !=-1
                return (sum(abs.(a1-a2).^p))^1/p
        end
end

function jaccardSimilarity(a1,a2)
        if errorCheck(a1,a2,1) !=-1
                return length(intersect(a1,a2))/length(union(a1,a2))
        end
end

function jaccardDistance(a1,a2)
        if errorCheck(a1,a2,1) !=-1
                return 1 - jaccardSimilarity(a1,a2)
        end
end


"""
    newton(f, fprime, x0, e, n)

Compute the root of the given function using Newton's method.
Uses the point at which tangent line intersects x-axis to find the next approximate of the root of function 'f',
as such it requires initial approximation `x0` and `fprime`, the derivative of function `f`.
Newton's method has very fast convergence provided a sufficiantly accurate initial approximation is chosen.

    newton(f::Function, fprime::Function, x0::Float64, e::Float64, n::Integer)

### OUTPUT:
  * `p` - approximation of the root of the function `f` (Float64) or,
  * `-1` - if `n` the maximum number of iterations was exceeded.
"""
function newton(f::Function, fprime::Function, x0::Float64, e::Float64, n::Integer)

    Afx = []

    for i in 1:n

        yprime = fprime(x0)

        if abs(yprime) < e
            warn("First derivative is zero")
            return x0
        end

        y = f(x0)
        x1 = x0 - y / yprime
        push!(Afx, f(x1))

        if abs(x1 - x0) < e
            return x1
        end

        x0 = x1
    end

    return -1
end


using NewtonsMethod
using Test

@testset "NewtonsMethod.jl" begin

# several @test for the root of a known function, given the f and analytical f' derivatives
f(x) = x - 10
f′(x)= 1
@test newtonroot(f,f′;x₀ = 0.0) == 10.0

f(x) = x-cos(x)
f′(x)= 1+sin(x)
@test newtonroot(f,f′;x₀ = 0.0) == 0.7390851332151607

f(x) = (x+5)^3
f′(x)= 3*(x+5)^2
@test newtonroot(f,f′;x₀ = 0.0) == -4.999999866018182

# tests of those roots using the automatic differentiation version of the function
f(x) = (x-1)^3
@test newtonroot(f;x₀=0.0) == 0.9999998643434097

f(x) = (x+1)^2
@test newtonroot(f;x₀=0.0) == -0.9999999403953552

f(x) = x^3 + x^2 + x - 1
@test newtonroot(f;x₀=0.0) == 0.5436890126920766

# test of finding those roots with a BigFloat and not just a Float64
f(x) = x - BigFloat(sqrt(5))
@test newtonroot(f;x₀ = 0.0) == BigFloat(sqrt(5))

f(x) =x-BigFloat(pi)
@test newtonroot(f;x₀ = 0.0) == BigFloat(pi)

# test of non-convergence for a function without a root (e.g. f(x)=2+x^2)
f(x) = x^2 + 2
@test newtonroot(f;x₀ = 1.0) == nothing

# test to ensure that the maxiter is working (e.g. what happens if you call maxiter = 5)
f(x) = (x-1)^3
f′(x)= 3*(x-1)^2
@test newtonroot(f,f′;x₀ = 0.0,maxiter = 5) == nothing

f(x) = (x-6)^6
@test newtonroot(f;x₀ = 0.0, maxiter = 10) == nothing

# test to ensure that tol is working
f(x) = (x-1)^3
f′(x)= 3*(x-1)^2
@test newtonroot(f,f′;x₀ = 0.0,tol = 1E-20) == 0.9999999999999999

f(x) = (x-6)^6
@test newtonroot(f;x₀ = 0.0,tol = 1E-3) == 5.995101279575065

end


# Theil.jl

###### theil #####
"""
    theil(v)

    Compute the Theil Index of a vector `v`.

# Examples
```julia
julia> using Inequality
julia> theil([8, 5, 1, 3, 5, 6, 7, 6, 3])
0.10494562214323544
```
"""
theil(v::AbstractVector{<:Real})::Float64 = sum(v .* log.(v/Statistics.mean(v))) / sum(v)

###### weighted theil #####
"""
    theil(v, w)

Compute the Theil Index of a vector `v`, using weights given by a weight vector `w`.

Weights must not be negative, missing or NaN. The weights and data vectors must have the same length.

# Examples
```julia
julia> using Inequality
julia> theil([8, 5, 1, 3, 5, 6, 7, 6, 3], collect(0.1:0.1:0.9))
0.08120013911680612
```
"""
function theil(v::AbstractVector{<:Real}, w::AbstractVector{<:Real})::Float64

    checks_weights(v, w)

    w = w[v .!= 0]
    v = v[v .!= 0]

    w = w/sum(w)
    v = v / sum(v .* w)
    return sum(v .* w .* log.(v))
end

function theil(v::AbstractVector{<:Real}, w::AbstractWeights)::Float64

    checks_weights(v, w)

    w = w[v .!= 0]
    v = v[v .!= 0]

    w = w/w.sum
    v = v / sum(v .* w)
    return sum(v .* w .* log.(v))
end

"""
    wtheil(v, w)

Compute the Theil Index of `v` with weights `w`. See also [`theil`](@theil)
"""
wtheil(v::AbstractVector{<:Real}, w::AbstractVector{<:Real})::Float64 = theil(v,w)

wtheil(v::AbstractVector{<:Real}, w::AbstractWeights)::Float64 = theil(v,w)


"""
newton(f,dfdx,x₁)

Use Newton's method to find a root of `f` starting from `x₁`, where
`dfdx` is the derivative of `f`. Returns a vector of root estimates.
"""
function newton(f,dfdx,x₁)
    # Operating parameters.
    funtol = 100*eps();  xtol = 100*eps();  maxiter = 40;

    x = [x₁]
    y = f(x₁)
    dx = Inf   # for initial pass below
    k = 1

    while (abs(dx) > xtol) && (abs(y) > funtol) && (k < maxiter)
        dydx = dfdx(x[k])
        dx = -y/dydx            # Newton step
        push!(x,x[k]+dx)        # append new estimate

        k += 1
        y = f(x[k])
    end

    if k==maxiter
        @warn "Maximum number of iterations reached."
    end

    return x
end

"""
secant(f,x₁,x₂)

Use the secant method to find a root of `f` starting from `x₁` and
`x₂`. Returns a vector of root estimates.
"""
function secant(f,x₁,x₂)
    # Operating parameters.
    funtol = 100*eps();  xtol = 100*eps();  maxiter = 40;

    x = [x₁,x₂]
    y₁ = f(x₁); y₂ = 100;
    dx = Inf   # for initial pass below
    k = 2

    while (abs(dx) > xtol) && (abs(y₂) > funtol) && (k < maxiter)
        y₂ = f(x[k])
        dx = -y₂ * (x[k]-x[k-1]) / (y₂-y₁)   # secant step
        push!(x,x[k]+dx)        # append new estimate

        k += 1
        y₁ = y₂    # current f-value becomes the old one next time
    end

    if k==maxiter
        @warn "Maximum number of iterations reached."
    end

    return x
end

"""
newtonsys(f,jac,x₁)

Use Newton's method to find a root of a system of equations,
starting from `x₁`. The functions `f` and `jac should return the
residual vector and the Jacobian matrix, respectively. Returns
history of root estimates as a vector of vectors.
"""
function newtonsys(f,jac,x₁)
    # Operating parameters.
    funtol = 1000*eps();  xtol = 1000*eps();  maxiter = 40;

    x = [float(x₁)]
    y,J = f(x₁),jac(x₁)
    dx = Inf   # for initial pass below
    k = 1

    while (norm(dx) > xtol) && (norm(y) > funtol) && (k < maxiter)
        dx = -(J\y)             # Newton step
        push!(x,x[k] + dx)    # append to history
        k += 1
        y,J = f(x[k]),jac(x[k])
    end

    if k==maxiter
        @warn "Maximum number of iterations reached."
    end

    return x
end

"""
fdjac(f,x0,y0)

Compute a finite-difference approximation of the Jacobian matrix for
`f` at `x0`, where `y0`=`f(x0)` is given.
"""
function fdjac(f,x0,y0)

δ = sqrt(eps())   # FD step size
m,n = length(y0),length(x0)
if n==1
    J = (f(x0+δ) - y0) / δ
else
    J = zeros(m,n)
    Iₙ = I(n)
    for j = 1:n
        J[:,j] = (f(x0 + δ*Iₙ[:,j]) - y0) / δ
    end
end

return J
end

"""
levenberg(f,x₁,tol)

Use Levenberg's quasi-Newton iteration to find a root of the system
`f`, starting from `x₁`, with `tol` as the stopping tolerance in
both step size and residual norm. Returns root estimates as a
matrix, one estimate per column.
"""
function levenberg(f,x₁,tol=1e-12)

# Operating parameters.
ftol = tol;  xtol = tol;  maxiter = 40;

x = zeros(length(x₁),maxiter)
x = [float(x₁)]
fₖ = f(x₁)
k = 1;  s = Inf;
Aₖ = fdjac(f,x₁,fₖ)   # start with FD Jacobian
jac_is_new = true

λ = 10;
while (norm(s) > xtol) && (norm(fₖ) > ftol) && (k < maxiter)
    # Compute the proposed step.
    B = Aₖ'*Aₖ + λ*I
    z = Aₖ'*fₖ
    s = -(B\z)

    xnew = x[k] + s
    fnew = f(xnew)

    # Do we accept the result?
    if norm(fnew) < norm(fₖ)    # accept
        y = fnew - fₖ
        push!(x,xnew)
        fₖ = fnew
        k += 1

        λ = λ/10   # get closer to Newton
        # Broyden update of the Jacobian.
        Aₖ = Aₖ + (y-Aₖ*s)*(s'/(s'*s))
        jac_is_new = false
    else                       # don't accept
        # Get closer to steepest descent.
        λ = 4λ
        # Re-initialize the Jacobian if it's out of date.
        if !jac_is_new
            Aₖ = fdjac(f,x[k],fₖ)
            jac_is_new = true
        end
    end
end

if (norm(fₖ) > 1e-3)
    @warn "Iteration did not find a root."
end

return x
end


#
###
# Lagrange polynomial matrices
###

"""
    Compute the Lagrange interpolation matrix from xi to xo.
    lagrange_poly_interp_mat(xₒ,xᵢ)
"""
function lagrange_interp_mat(xo,xi)

    no = length(xo)
    ni = length(xi)

    a = ones(1,ni)
    for i=1:ni
        for j=1:(i-1)  a[i]=a[i]*(xi[i]-xi[j]); end
        for j=(i+1):ni a[i]=a[i]*(xi[i]-xi[j]); end
    end
    a = 1 ./ a # Barycentric weights

    J = zeros(no,ni)
    s = ones(1,ni)
    t = ones(1,ni)
    for i=1:no
        x = xo[i]
        for j=2:ni
            s[j]      = s[j-1]    * (x-xi[j-1]   )
            t[ni+1-j] = t[ni+2-j] * (x-xi[ni+2-j])
        end
        J[i,:] = a .* s .* t
    end

    return J
end

"""
 Compute derivative matrix for lagrange
 interpolants on points [x]
"""
function lagrange_deriv_mat(x)

    n = length(x)

    a = ones(1,n)
    for i=1:n
        for j=1:(i-1) a[i]=a[i]*(x[i]-x[j]) end
        for j=(i+1):n a[i]=a[i]*(x[i]-x[j]) end
    end
    a = 1 ./ a # Barycentric weights

    # diagonal elements
    D = x .- x'
    for i=1:n D[i,i] = 1. end
    D = 1 ./ D
    for i=1:n
        D[i,i] = 0.
        D[i,i] = sum(D[i,:])
    end

    # off-diagonal elements
    for j=1:n for i=1:n
        if(i!=j) D[i,j] = a[j] / (a[i]*(x[i]-x[j])) end
    end end

    return D
end


################################################################################

"""
    euclDist(arr::Array{T, 1}, h::Array{T, 1}) where {T <: Number}

# Description
Euclidean distance.

# Examples
```
julia> euclDist(collect(1:10), fill(5, 10))
9.219544457292887
```

See also: [`bhattDist`](@ref), [`amplitude`](@ref)
"""
function euclDist(arr::Array{T,1}, h::Array{T,1}) where {T <: Number}
  return (arr .- h) .^ 2 |> sum |> sqrt
end

"""
    bhattDist(arr::Array{T, 1}, h::Array{T, 1}) where {T <: Number}

# Description
Bhattacharyya distance.

# Examples
```
julia> bhattDist(collect(1:10), fill(5, 10))
-3.936532135073928
```

See also: [`euclDist`](@ref), [`amplitude`](@ref)
"""
function bhattDist(arr::Array{T,1}, h::Array{T,1}) where {T <: Number}
  return -log((arr .* h .|> sqrt |> sum) + 1)
end

"""
    amplitude(arr::Array{T, 1}) where {T <: Number}

# Description
Amplitude.

# Examples
```
julia> amplitude(collect(1:10))
19.621416870348583
```

See also: [`euclDist`](@ref), [`bhattDist`](@ref)
"""
function amplitude(arr::Array{T,1}) where {T <: Number}
  return arr .^ 2 |> sum |> sqrt
end

################################################################################


export eye

"""
`eye(T,n)` returns an `n`-by-`n` identity matrix whose entries
are numbers of type `T` (`Float64` by default).
"""
function eye(T,n)
    A = zeros(T,n,n)
    for j=1:n
        A[j,j] = 1
    end
    return A
end

eye(n) = eye(Float64,n)


# Continuation of ex 7.13, Least Squares WRM

using NumericalMethodsforEngineers
using Test

@sym begin
  ClearAll(xi, yi, N1, Y, Ydotdot, C11, ytilde1, ytilde2)
  xi = [0, 1//3, 2//3, 1]
  yi = [0, a, b, 1]
  Y(x_) := LagrangePolynomial(xi, yi)
  #
  # Can be re-formulated as Y(x) = F(x) + C1(a) * Ψ1(x) + C2(a) * Ψ2(x)
  #
  # F(x_) := (1/2)*(9x^3 - 9x^2 + 2x)
  # C1 = (27/2)*(a-b)
  Ψ1(x_) := x^3 - x^2
  # C2 = -(9/2)*(2a-b)
  Ψ2(x_) := x^2 - x
  #
  # Y(x_) := F(x) + C1(a)*Ψ1(x) + C2(a)*Ψ2(x)
  #
  Ydotdot(x_) = D(Y(x), x, 2)
  R(x_) := Ydotdot(x) - 3*x - 4*Y(x)
  Rdota(x_) = D(R(x), a)
  Rdotb(x_) = D(R(x), b)
  sol = Solve([Integrate(R(x)*Rdota(x), [x, 0, 1]), Integrate(R(x)*Rdotb(x), [x, 0, 1])], [a, b])
  SetJ(r, ToString(Simplify(R(x))))
  SetJ(a, ToString(sol[1][1][2]))
  SetJ(b, ToString(sol[1][2][2]))
  ytilde1 = Simplify(Y(x) ./ (a => sol[1][1][2]))
  ytilde2 = Simplify(ytilde1 ./ (b => sol[1][2][2]))
  SetJ(y, ToString(Simplify(Expand(ytilde2))));
end

println("\n\nExample 7.17b: y'' = 3x + 4y, y(0)=0, y(1)=1")
println("by 2-point least squares Weighted Residual Method")
@sym Println("\nY(x) = $(Simplify(Expand(Y(x))))\n")
@sym Println("R(x) = $(R(x))\n")
@eval a = $(Meta.parse(a))
@eval b = $(Meta.parse(b))
println("(a,b) = $((a, b))\n")
println("C1 = $((27/2*(a-b)))\n")
println("C2 = $(-(9/2*(2a-b)))\n")
@eval rf_2pt_leastsquares(x, a, b) = $(Meta.parse(r))
@eval ytilde_2pt_leastsquares(x) = $(Meta.parse(y))
println()

rf_2pt_leastsquares_1(x) = rf_2pt_leastsquares(x, a, b)
println()

@test r == "-9 - 45a + 36b + 20x - 63b*x + 45a*x + 18*x^2 - 72b*x^2 + 90a*x^2 - 18x^3 - 54a*x^3 + 54b*x^3"
@test y == "(1/9693)*x*(2420 - 3311x + 10584*x^2)"
#@test (quadgk(rf_2pt_leastsquares_1, 0, 1))[1] < 5*eps()


#!/usr/bin/env julia

"""
# Problem 1: Multiples of 3 and 5

If we list all the natural numbers below 10 that are multiples of 3 or 5, we
get 3, 5, 6 and 9. The sum of these multiples is 23.

Find the sum of all the multiples of `m` or `n` below `limit`.
"""
module Problem001

include("Sequences.jl")

"""
    solve(; m::Integer = 3, n::Integer = 5, limit::Integer = 1000)

Returns the solution for [`Problem001`](@ref) with the given parameters.

## Preconditions
- `m > 0`
- `n > 0`
- `limit > 0`
"""
function solve(; m::Integer = 3, n::Integer = 5, limit::Integer = 1000)
    m_sum = sum_divisible_by(m, limit)
    n_sum = sum_divisible_by(n, limit)
    lcm_sum = sum_divisible_by(lcm(m, n), limit)
    return m_sum + n_sum - lcm_sum
end

"""
    sum_divisible_by(n::Integer, limit::Integer)

Returns the sum of natural numbers below `limit` that are divisible by `n`.

## Preconditions
- `n > 0`
- `limit > 0`

## Examples
```jldoctest
julia> sum_divisible_by(3, 10)
18

julia> sum_divisible_by(5, 10)
5

julia> sum_divisible_by(5, 5)
0
```
"""
function sum_divisible_by(n::Integer, limit::Integer)
    return Sequences.arithmetic_series(n, (limit - 1) ÷ n, n)
end

end