using StanVariational
using Test

bernoulli_model = "
data {
  int<lower=1> N;
  int<lower=0,upper=1> y[N];
}
parameters {
  real<lower=0,upper=1> theta;
}
model {
  theta ~ beta(1,1);
  y ~ bernoulli(theta);
}
";

bernoulli_data = Dict("N" => 10, "y" => [0, 1, 0, 1, 0, 0, 0, 0, 0, 1])

stanmodel = VariationalModel("bernoulli", bernoulli_model)

(sample_file, log_file) = stan_variational(stanmodel; data=bernoulli_data)

if sample_file !== Nothing

  @testset "Bernoulli variational example" begin
    # Read sample summary (in ChainDataFrame format)
    sdf = read_summary(stanmodel)

    @test sdf[:theta, :mean][1] ≈ 0.32 atol=0.1
  end

end


module MathUtils

export log_sum_exp
function log_sum_exp(logs)
  maxlog = maximum(logs)
  return maxlog + log(sum(exp(logs - maxlog)))
end

export sample_discrete
function sample_discrete(dist)
  r = rand()
  s = 0.
  for i=1:length(dist)
    s += dist[i]
    if r <= s
      return(i)
    end
  end
  return(length(dist))
end

export sample_discrete_logspace
function sample_discrete_logspace(log_unnormalized_dist)
  n = length(log_unnormalized_dist)
  logsum = log_sum_exp(log_unnormalized_dist)
  logr = log(rand())
  logc = log_unnormalized_dist[1] - logsum
  for i=1:n-1
    if logr <= logc
      return i
    end
    logp = log_unnormalized_dist[i+1] - logsum
    if logp > logc
      logc = logp + log1p(exp(logc - logp))
    else
      logc = logc + log1p(exp(logp - logc))
    end
  end
  return(n)
end

export normalize_log_dist
function normalize_log_dist(d)
  d - log_sum_exp(d)
end

export is_valid_dist
function is_valid_dist(dist)
  abs(sum(dist) - 1.0) <= eps(1.0) && minimum(dist) >= 0.0
end

export importance_sample
function importance_sample(π, q, n)
  assert(length(π) == length(q))
  assert(is_valid_dist(π))
  assert(is_valid_dist(q))
  p = zeros(length(π))
  counts = zeros(length(π))
  for i=1:n
    x = sample_discrete(q)
    p[x] += (1.0 / n) * π[x] / q[x]
    counts[x] += 1
  end
  p, counts
end

end # module


# https://www.sciencedirect.com/science/article/pii/S0895717710005935

function boxMuller(u,v)
    r = sqrt(-2 * log(u))
    θ = 2π*v
    (r * cos(θ), r * sin(θ))
end

normals() = ()
normals(x::Vector) = normals(x...)
normals(x::Tuple) = normals(x...)
normals(u,v,args...) = (boxMuller(u,v)..., normals(args...)...)
normals(u) = norminvcdf(u)

export StdNormals
export Normals

StdNormals(N) = Quasi(N,normals)

Normals(N,μ,σ) = map(x -> σ*x + μ, StdNormals(N))

# MvNormals(X)


"""
    FBMKernel(; h::Real=0.5)

Fractional Brownian motion kernel with Hurst index `h` from (0,1) given by
```
    κ(x,y) =  ( |x|²ʰ + |y|²ʰ - |x-y|²ʰ ) / 2
```

For `h=1/2`, this is the Wiener Kernel, for `h>1/2`, the increments are
positively correlated and for `h<1/2` the increments are negatively correlated.
"""
struct FBMKernel{T<:Real} <: Kernel
    h::Vector{T}
    function FBMKernel(; h::T=0.5) where {T<:Real}
        @assert 0.0 <= h <= 1.0 "FBMKernel: Given Hurst index h is invalid."
        return new{T}([h])
    end
end

@functor FBMKernel

function (κ::FBMKernel)(x::AbstractVector{<:Real}, y::AbstractVector{<:Real})
    modX = sum(abs2, x)
    modY = sum(abs2, y)
    modXY = sqeuclidean(x, y)
    h = first(κ.h)
    return (modX^h + modY^h - modXY^h)/2
end

(κ::FBMKernel)(x::Real, y::Real) = (abs2(x)^first(κ.h) + abs2(y)^first(κ.h) - abs2(x - y)^first(κ.h)) / 2

Base.show(io::IO, κ::FBMKernel) = print(io, "Fractional Brownian Motion Kernel (h = ", first(κ.h), ")")

_fbm(modX, modY, modXY, h) = (modX^h + modY^h - modXY^h)/2

_mod(x::AbstractVector{<:Real}) = abs2.(x)
_mod(x::ColVecs) = vec(sum(abs2, x.X; dims=1))
_mod(x::RowVecs) = vec(sum(abs2, x.X; dims=2))

function kernelmatrix(κ::FBMKernel, x::AbstractVector)
    modx = _mod(x)
    modxx = pairwise(SqEuclidean(), x)
    return _fbm.(modx, modx', modxx, κ.h)
end

function kernelmatrix!(K::AbstractMatrix, κ::FBMKernel, x::AbstractVector)
    modx = _mod(x)
    pairwise!(K, SqEuclidean(), x)
    K .= _fbm.(modx, modx', K, κ.h)
    return K
end

function kernelmatrix(κ::FBMKernel, x::AbstractVector, y::AbstractVector)
    modxy = pairwise(SqEuclidean(), x, y)
    return _fbm.(_mod(x), _mod(y)', modxy, κ.h)
end

function kernelmatrix!(
    K::AbstractMatrix,
    κ::FBMKernel,
    x::AbstractVector,
    y::AbstractVector,
)
    pairwise!(K, SqEuclidean(), x, y)
    K .= _fbm.(_mod(x), _mod(y)', K, κ.h)
    return K
end


### A Pluto.jl notebook ###
# v0.12.20

using Markdown
using InteractiveUtils

# ╔═╡ b2937383-dcbe-4863-9267-eca115a7b666
begin
	using Pkg
	Pkg.activate(mktempdir())
	Pkg.Registry.update()
	Pkg.add("Yao")
	Pkg.add("YaoPlots")
end

# ╔═╡ 57209cca-1ede-11eb-283b-196a50e6f1eb
using Yao,YaoPlots

# ╔═╡ 76794562-1ed0-11eb-18ec-ad4020344ce6
md"# Arithmetic using qubits"

# ╔═╡ ad61776c-1ed9-11eb-20af-7d41b6e281c3
md"## Binary addition"

# ╔═╡ 949416ee-1ed0-11eb-2a86-dd52c9b1d5f9
md"Suppose we've to add two numbers in binary form! Say... `` 5 `` and `` 7 ``."

# ╔═╡ aad0de2e-1ed0-11eb-3efc-95c0072fff76
md"In binary form, `` 5 `` can be represented by `` 101 `` and `` 7 `` can be represented by `` 111 ``. Remember when adding two numbers in binary form,

- `` 0 + 0 = 0 ``
- `` 0 + 1 = 1 ``
- `` 1 + 0 = 1 ``
- `` 1 + 1 = 0 `` with a carry of `` 1 ``

So adding `` 5 `` and `` 7 `` in binary form looks somewhat like this.
- `` \;\;\;\; ₁ \;₁\;\;₁\;₀ ``  ← **Carry**
- ``  \;\;\;\;\;\;1\; 0\; 1 ``
- `` \;\; `` + `` \; 1\; 1\; 1 ``
- `` \;\; `` _________
- `` \;\;\;1\;1\;0\;0 ``

Starting from right, and moving to left

- `` 1 + 1 = 0 `` with a carry of `` 1 ``
- `` 0 + 1 = 1 `` which is added to the carried value, which is `` 1 ``, so `` 1 + 1 = 0 `` with a carry of `` 1 ``, again
- `` 1 + 1 = 0 `` with a carry of `` 1 ``. The result is added again to the carried value, which was `` 1 ``, so `` 0 + 1 = 1 ``
- The remaining carried value, which is `` 1 ``, is put as it is.
"

# ╔═╡ c731fcb8-1ed7-11eb-2800-4fda7d5ae5e7
md"The result is `` \; [( 1 × 2^3 ) + ( 1 × 2^2 ) + ( 0 × 2^1 ) + ( 0 × 2^0 )] = 12. ``"

# ╔═╡ b914c0b4-1ed9-11eb-37e9-8f1a60dbfcb2
md"#### Quantum addition circuit for one pair of qubits"

# ╔═╡ ec622eb0-1eda-11eb-0ca2-1bc4b4f1a730
md"We'll try making the adder circuit for addition of two numbers  `` (0-7) ``. We'll need `` 3 `` qubits to represent each of the two numbers, which means a total of `` 6 `` qubits."

# ╔═╡ 30953b42-1ede-11eb-32ee-af55afa9221f
begin
	FirstNumber = ArrayReg(bit"111") #The first number is 7
	SecondNumber = ArrayReg(bit"101") #The second number is 5
end

# ╔═╡ 2ae7c4ec-1ee0-11eb-1c46-59d4d3084b39
md"For adding each pair of qubits, we'll need two more qubits to hold the carried in value and carried out value. Since the carry out of the 1st pair acts as the carry in of second pair, the carry out of the second pair acts as the carry in of the 3rd pair, we need 4 more qubits for carry in and carry out, with

- 1 qubit for the carry-in of first pair
- 1 qubit for the carry-out of first pair and the carry-in of the second pair
- 1 qubit for the carry-out of second pair and the carry-in of the third pair
- 1 qubit for the carry-out of third pair

In total, we require a total of `` 10 `` qubits to add two numbers in the range `` \; 0-7 ``."

# ╔═╡ 1ac1e8c0-1f2d-11eb-1cc5-5986d94a86c2
md"So lets try making the quantum circuit for adding two qubits. We need a qubit for carry-in, which will be zero for the rightmost pair, and a qubit for carry-out, which will act as the carry in for the next pair"

# ╔═╡ 1d8ce7a2-1f2e-11eb-1b74-0d601958a2c9
md"Consider this circuit"

# ╔═╡ 802cf394-1f2d-11eb-23ad-95c475382371
begin
	OneqbQFA = chain(4, control(2:3, 4=>X), control(2, 3=>X), control([1 3], 4=>X), control(1, 3=>X))
	plot(OneqbQFA)
end

# ╔═╡ 36613c90-1f2e-11eb-1f5e-d3e079a8deda
md"
- The top qubit holds the carry-in value.
- The bottom qubit has the state `` |0〉 ``.
- The 2nd and the 3rd qubits hold the pair to be added.
After passing throught the circuit,
- The top qubit holds the carry-in value.
- The bottom qubit holds the carry-out value.
- The 2nd qubit is as it was, while the 3rd qubit holds the addition of the 2nd and 3rd qubit.
"

# ╔═╡ 88d976e2-1f2e-11eb-0b7c-ad9557d124e6
md"### The use of CX gate in Arithmetics"

# ╔═╡ ade4a628-1f2e-11eb-32d0-c16546a16e44
md"Consider this circuit"

# ╔═╡ b307d638-1f2f-11eb-19ec-5fb58b49ff0f
begin
	a = chain(2, control(1, 2=>X))
	plot(a)
end

# ╔═╡ 31a37e02-1f30-11eb-3c16-494e05427f33
md"Lets try passing different values through this circuit"

# ╔═╡ 41c223ec-1f30-11eb-038b-efd7279954e6
measure((ArrayReg(bit"00") |> a)) #The output is 00, when we passed 00

# ╔═╡ 6ba13306-1f30-11eb-04c7-9f1c51a49915
measure(ArrayReg(bit"01") |> a) #The output is 11, when we passed 01
#Remember, the circuit takes the qubits as inputs, in reverse order. The rightmost qubit is the 1st qubit here, and since its |1〉, the 2nd qubit gets flipped to |1〉 too.

# ╔═╡ 95dbc5be-1f30-11eb-232f-e72c7bfbda61
measure(ArrayReg(bit"10") |> a) #The output is 10, when we passed 10
#Remember, the circuit takes the qubits as inputs, in reverse order. The rightmost qubit is the 1st qubit here, and since its |0〉, the 2nd qubit is left untouched.

# ╔═╡ d33faf76-1f30-11eb-0dd7-3d1331633c0a
measure(ArrayReg(bit"11") |> a) #The output is 01, when we passed 11
#Remember, the circuit takes the qubits as inputs, in reverse order. The rightmost qubit is the 1st qubit here, and since its |1〉, the 2nd qubit is flipped to |0〉.

# ╔═╡ b3823ea8-1f31-11eb-0651-d55953af774a
md"Lets analyze the output
- When the top qubit is `` |0〉 `` and the bottom qubit is `` |0〉 ``, the bottom qubit is left untouched.
- When the top qubit is `` |1〉 `` and the bottom qubit is `` |0〉 ``, the bottom qubit is flipped to `` |1〉 ``.
- When the top qubit is `` |0〉 `` and the bottom qubit is `` |1〉 ``, the bottom qubit is left untouched.
- When the top qubit is `` |1〉 `` and the bottom qubit is `` |1〉 ``, the bottom qubit is flipped to `` |0〉 ``.
Compare this with,
- `` 0 + 0 = 0 ``
- `` 0 + 1 = 1 ``
- `` 1 + 0 = 1 ``
- `` 1 + 1 = 0 ``
In all the cases, the bottom qubit holds the added value.

How about adding 3 qubits?
"

# ╔═╡ c7b7511c-1f34-11eb-37ba-f71851aceb94
begin
	b = chain(3, control(2, 3=>X), control(1, 3=>X))
	plot(b)
end

# ╔═╡ fce93e40-1f34-11eb-3ea6-a11d129d7af3
measure.([(ArrayReg(bit"000") |> b)
		(ArrayReg(bit"001") |> b)
		(ArrayReg(bit"010") |> b)
		(ArrayReg(bit"011") |> b)
		(ArrayReg(bit"110") |> b)
		(ArrayReg(bit"101") |> b)
		(ArrayReg(bit"100") |> b)
		(ArrayReg(bit"111") |> b)])
#If the output is not visible, click on the output to expand it.

# ╔═╡ 1604f776-1f36-11eb-03fc-c5575a528776
md"The toffoli gate acts as the carry-out. If both input qubits are `` |1〉 ``, the 3rd qubit is flipped."

# ╔═╡ ba41d664-1f3a-11eb-2334-83fa55bc3bb6
md"### The Quantum Adder Circuit"

# ╔═╡ f4db1d44-1f3a-11eb-0b8c-4589d745d435
md"We already made the circuit for adding two qubits. Remember that the carry-out for the first pair, becomes the carry in for the second pair. With this, here's the circuit for a quantum adder for 2 numbers between `` 0-7 ``."

# ╔═╡ 92b2e1b4-4457-11eb-22c0-7b5173d6bd97
plot(chain(10, put(1:4=>label(chain(4), "Full adder\nfor a pair\n of qubits")), put(4:7=>label(chain(4), "Full adder\nfor a pair\n of qubits")), put(7:10=>label(chain(4), "Full adder\nfor a pair\n of qubits"))))

# ╔═╡ 71d66a6e-4458-11eb-0a56-69919a6dbdca
md"Below is a complete circuit for quantum adder for 2 numbers between `` 0 - 7 ``."

# ╔═╡ 412203a2-1f3b-11eb-3de7-45f7eccd697e
begin
	nqbQFA = chain(10, put(1:4=>OneqbQFA), put(4:7=>OneqbQFA), put(7:10=>OneqbQFA))
	plot(nqbQFA)
end

# ╔═╡ c2afc0f8-1f3b-11eb-0626-c5120a452e38
md"The input to the circuit being - "

# ╔═╡ e4b012f0-1f3b-11eb-1336-af9ba42a98c8
begin
	input = join(zero_state(4), FirstNumber, SecondNumber)
	# 10 9 8 7 6 5 4  3 2 1
	# 1  0 1 1 1 1 0  0 0 0
	# 2  5 8 3 6 9 1  4 7 10
	reorder!(input, [2 5 8 3 6 9 1 4 7 10]) #Re-ordering the qubits, for being the input to the above circuit
end

# ╔═╡ 1abc9e96-4b70-11eb-1661-7f37fdea9130
measure(input)

# ╔═╡ 04c694e4-1f3c-11eb-1faf-0d08d391ce64
results = input |> nqbQFA |> r->measure(r, nshots=1024)

# ╔═╡ 1083740a-1f3c-11eb-2950-c56e4f20087c
stringres = reverse(string(Int(results[1]), base=2, pad=10)) #To convert it to string

# ╔═╡ 23e71498-1f3c-11eb-1c3e-2dac1694afeb
md"Remember, the 3, 6 and 9th qubits hold the added values. The 10 qubit holds the final carry-out, if any."

# ╔═╡ 4e89800a-1f3c-11eb-239f-f92807c81881
output = parse(Int64, reverse(stringres[3] * stringres[6] * stringres[9] * stringres[10]), base=2)

# ╔═╡ 270619d0-1f50-11eb-3a89-1187e12bd4f9
md"## Quantum subtractor"

# ╔═╡ 976e00e8-1f50-11eb-0d93-77e2ba841a09
md"Binary subtraction has a borrow instead of carry. The rules of binary subtraction look somewhat like this.
- `` 0 - 0 = 0 ``
- `` 0 - 1 = 1 `` with a borrow of `` 1 ``
- `` 1 - 0 = 1 ``
- `` 1 - 1 = 0 ``
"

# ╔═╡ 346af906-1f50-11eb-3286-dbf3014c8fab
begin
	OneqbSubtractor = chain(4, control(1, 3=>X), put(2=>X), control(2:3, 4=>X), control(1, 3=>X), control([1 3], 4=>X), put(2=>X), control(2, 3=>X), control(1, 3=>X))
	plot(OneqbSubtractor)
end

# ╔═╡ e2659b46-1f51-11eb-34db-dbc917b977dd
begin
	nqbQFS = chain(10, put(1:4=>OneqbSubtractor), put(4:7=>OneqbSubtractor), put(7:10=>OneqbSubtractor))
	plot(nqbQFS)
end

# ╔═╡ adeee882-1f55-11eb-27a4-d79963776e20
measure(join(zero_state(1), ArrayReg(bit"10"), zero_state(1)) |> OneqbSubtractor)

# ╔═╡ 2ff67ae4-4b4c-11eb-17a6-45e64c916c48
begin
	p = ArrayReg(bit"111")
	q = ArrayReg(bit"101")
	inp = join(zero_state(4), p, q)
	reorder!(inp, [3 6 9 2 5 8 1 4 7 10])
end

# ╔═╡ 31eb23f6-4b5e-11eb-2aa3-39ef184edb69
measure(inp)

# ╔═╡ 1f828098-1f52-11eb-1d00-27f10270a814
result = inp |> nqbQFS |> r->measure(r, nshots=1024)

# ╔═╡ 3cbd0f52-1f52-11eb-06f4-ffe29cccde57
stringsub = reverse(string(Int(result[1]), base=2, pad=10)) #To convert it to string

# ╔═╡ 658eec52-1f52-11eb-39aa-bbf1fcfb3517
out =  parse(Int64, reverse(stringsub[3] * stringsub[6] * stringsub[9] * stringsub[10]), base=2)

# ╔═╡ ffd40492-1f56-11eb-210d-ef6bf293244e
md"This circuit only subtracts numbers if the answer is expected to be positive. It can't solve for calculations like `` 5 - 6 = -1 ``."

# ╔═╡ Cell order:
# ╠═b2937383-dcbe-4863-9267-eca115a7b666
# ╟─76794562-1ed0-11eb-18ec-ad4020344ce6
# ╟─ad61776c-1ed9-11eb-20af-7d41b6e281c3
# ╟─949416ee-1ed0-11eb-2a86-dd52c9b1d5f9
# ╟─aad0de2e-1ed0-11eb-3efc-95c0072fff76
# ╟─c731fcb8-1ed7-11eb-2800-4fda7d5ae5e7
# ╟─b914c0b4-1ed9-11eb-37e9-8f1a60dbfcb2
# ╟─ec622eb0-1eda-11eb-0ca2-1bc4b4f1a730
# ╠═57209cca-1ede-11eb-283b-196a50e6f1eb
# ╠═30953b42-1ede-11eb-32ee-af55afa9221f
# ╟─2ae7c4ec-1ee0-11eb-1c46-59d4d3084b39
# ╟─1ac1e8c0-1f2d-11eb-1cc5-5986d94a86c2
# ╟─1d8ce7a2-1f2e-11eb-1b74-0d601958a2c9
# ╠═802cf394-1f2d-11eb-23ad-95c475382371
# ╟─36613c90-1f2e-11eb-1f5e-d3e079a8deda
# ╟─88d976e2-1f2e-11eb-0b7c-ad9557d124e6
# ╟─ade4a628-1f2e-11eb-32d0-c16546a16e44
# ╠═b307d638-1f2f-11eb-19ec-5fb58b49ff0f
# ╟─31a37e02-1f30-11eb-3c16-494e05427f33
# ╠═41c223ec-1f30-11eb-038b-efd7279954e6
# ╠═6ba13306-1f30-11eb-04c7-9f1c51a49915
# ╠═95dbc5be-1f30-11eb-232f-e72c7bfbda61
# ╠═d33faf76-1f30-11eb-0dd7-3d1331633c0a
# ╟─b3823ea8-1f31-11eb-0651-d55953af774a
# ╠═c7b7511c-1f34-11eb-37ba-f71851aceb94
# ╠═fce93e40-1f34-11eb-3ea6-a11d129d7af3
# ╟─1604f776-1f36-11eb-03fc-c5575a528776
# ╟─ba41d664-1f3a-11eb-2334-83fa55bc3bb6
# ╟─f4db1d44-1f3a-11eb-0b8c-4589d745d435
# ╟─92b2e1b4-4457-11eb-22c0-7b5173d6bd97
# ╟─71d66a6e-4458-11eb-0a56-69919a6dbdca
# ╠═412203a2-1f3b-11eb-3de7-45f7eccd697e
# ╟─c2afc0f8-1f3b-11eb-0626-c5120a452e38
# ╠═e4b012f0-1f3b-11eb-1336-af9ba42a98c8
# ╠═1abc9e96-4b70-11eb-1661-7f37fdea9130
# ╠═04c694e4-1f3c-11eb-1faf-0d08d391ce64
# ╠═1083740a-1f3c-11eb-2950-c56e4f20087c
# ╟─23e71498-1f3c-11eb-1c3e-2dac1694afeb
# ╠═4e89800a-1f3c-11eb-239f-f92807c81881
# ╟─270619d0-1f50-11eb-3a89-1187e12bd4f9
# ╟─976e00e8-1f50-11eb-0d93-77e2ba841a09
# ╠═346af906-1f50-11eb-3286-dbf3014c8fab
# ╠═e2659b46-1f51-11eb-34db-dbc917b977dd
# ╠═adeee882-1f55-11eb-27a4-d79963776e20
# ╠═2ff67ae4-4b4c-11eb-17a6-45e64c916c48
# ╠═31eb23f6-4b5e-11eb-2aa3-39ef184edb69
# ╠═1f828098-1f52-11eb-1d00-27f10270a814
# ╠═3cbd0f52-1f52-11eb-06f4-ffe29cccde57
# ╠═658eec52-1f52-11eb-39aa-bbf1fcfb3517
# ╟─ffd40492-1f56-11eb-210d-ef6bf293244e


# coding: utf-8

# In[1]:

#coin change is a straight forward dynamic programming problem
#you are given one note and a set of coins of different values
#assuming you have infinite supply of coins of different values
#we need to compute different ways of making change
#the order of the coins doesnt matter in this case
#more details can be found in the following link
# https://www.geeksforgeeks.org/coin-change-dp-7/
#the script can also be done in dynamic programming
# https://github.com/je-suis-tm/recursion-and-dynamic-programming/blob/master/coin%20change%20dynamic%20programming.jl

# In[2]:

#to solve this problem via recursion
#we divide one big problem into two sub problems
#the case where coin of η value is excluded in the solutions
#and the case where at least one coin of η value is included
function coin_change(num,choice)

    #base case
    if num==0

        return 1

    end

    #prevent stack overflow
    if num<0

        return 0

    end

    output=0

    #iterate through cases of exclusion
    for i in 1:length(choice)

        #case of inclusion
        include=coin_change(num-choice[i],choice)
        exclude=0

        #case of exclusion
        if i>=2

            exclude=coin_change(num,choice[1:(i-1)])

        end

        #two sub problems merge into a big one
        output=include+exclude

    end

    return output

end

# In[3]:

coin_change(10,[1,2,3])


#!/usr/bin/env julia
# -*- coding: utf-8 -*-
# File   : circle_approx.jl
# License: MIT
# Author : Andrei Leonard Nicusan <a.l.nicusan@bham.ac.uk>
# Date   : 21.05.2021

using GLMakie
using LazySets

# Return points on a circle
function circle(;centre = [0., 0.], radius = 1., n = 16)
    points = zeros(2, n + 1)
    for (i, α) in enumerate(LinRange(0, 2pi, n + 1))
        points[1, i] = centre[1] + radius * cos(α)
        points[2, i] = centre[2] + radius * sin(α)
    end

    points
end

# Analytical circle points
t = LinRange(0, 2pi, 100)
circle_analytical = [cos.(t) sin.(t)]

# Ploting
fig = Figure()
axes = [Axis(fig[1, i]) for i in 1:2]
for ax in axes
    ax.aspect = AxisAspect(1)
end

lines!(axes[1], circle_analytical[:, 1], circle_analytical[:, 2], color = :red, linewidth = 3,
       label = "∞")

# Plot polygonal approximations and area ratio
nverts = collect(4:64)
ratios = zeros(length(nverts))
to_plot = [4, 8, 16, 32]

for (i, n) in enumerate(nverts)
    circle_points = circle(n = Int64(n))
    circle_poly = VPolygon(circle_points)
    ratios[i] = area(circle_poly) / pi

    if n in to_plot
        lines!(axes[1], circle_points[1, :], circle_points[2, :], label = "$n")
    end
end

lines!(axes[2], nverts, ratios)

axislegend(axes[1])
fig


# Load X and y variable
using JLD
data = load("quantum.jld")
(X,y) = (data["X"],data["y"])

# Add bias variable, initialize w, set regularization and optimization parameters
(n,d) = size(X)
lambda = 1

# Initialize
maxPasses = 10
progTol = 1e-4
verbose = true
w = zeros(d,1)
lambda_i = lambda/n # Regularization for individual example in expectation

# Start running stochastic gradient
w_old = copy(w);
for k in 1:maxPasses*n

    # Choose example to update 'i'
    i = rand(1:n)

    # Compute gradient for example 'i'
    r_i = -y[i]/(1+exp(y[i]*dot(w,X[i,:])))
    g_i = r_i*X[i,:] + (lambda_i)*w

    # Choose the step-size
    alpha = 1/(lambda_i*k)

    # Take thes stochastic gradient step
    w -= alpha*g_i

    # Check for lack of progress after each "pass"
    if mod(k,n) == 0
        yXw = y.*(X*w)
        f = sum(log.(1 + exp.(-y.*(X*w)))) + (lambda/2)norm(w)^2
        delta = norm(w-w_old,Inf);
        if verbose
            @printf("Passes = %d, function = %.4e, change = %.4f\n",k/n,f,delta);
        end
        if delta < progTol
            @printf("Parameters changed by less than progTol on pass\n");
            break;
        end
        w_old = copy(w);
    end
end


#############################################################################
#  Copyright 2016, Iain Dunning, Joey Huchette, Miles Lubin, and contributors
#  This Source Code Form is subject to the terms of the Mozilla Public
#  License, v. 2.0. If a copy of the MPL was not distributed with this
#  file, You can obtain one at http://mozilla.org/MPL/2.0/.
#
# Author: Louis Luangkesorn
# Date: Jan 30, 2015
#
# Allocation of passenger cars to trains to minimize cars required or car-miles run.
# Based on
#   Fourer, D.M. Gay and Brian W. Kernighan, A Modeling Language for Mathematical
#   Programming, http://www.ampl.com/REFS/amplmod.ps.gz
#   Appendix D
#############################################################################
using JuMP

ORIG = ["GARY", "CLEV", "PITT"];
DEST = ["FRA", "DET", "LAN", "WIN", "STL", "FRE", "LAF"];

supply = [1400 2600 2900];
demand = [ 900 1200  600 400 1700 1100 1000];

@assert sum(supply) == sum(demand);

cost = [
39   14   11   14   16   82    8;
27    9   12    9   26   95   17;
24   14   17   13   28   99   20
]

m = Model();

@variable(m, Trans[i=1:length(ORIG), j=1:length(DEST)] >= 0);

@objective(m, Min, sum{cost[i,j] * Trans[i,j], i=1:length(ORIG), j=1:length(DEST)});

@constraint(m, xyconstr[i=1:1:length(ORIG)], sum{Trans[i,j], j=1:length(DEST)} == supply[i]);

@constraint(m, xyconstr[j = 1:length(DEST)], sum{Trans[i,j], i=1:length(ORIG)} == demand[j]);

println("Solving original problem...")
status = solve(m);

if status == :Optimal
	@printf("Optimal!\n");
	@printf("Objective value: %d\n", getobjectivevalue(m));
	@printf("Transpotation:\n");
	for j = 1:length(DEST)
		@printf("\t%s", DEST[j]);
	end
	@printf("\n");
	for i = 1:length(ORIG)
		@printf("%s", ORIG[i]);
		for j = 1:length(DEST)
			@printf("\t%d", getvalue(Trans[i,j]));
		end
		@printf("\n");
	end
else
    @printf("No solution\n");
end


# Interpolates the x, y points to estimate the value at the point "t".
# val (out) ...... the estimated function value at the point "t".
# t (in).......... the "x" value at which to estimate the value
# X (array, in) .. the x-values of the points
# Y (array, in) .. the y-values of the points
# n (in) ......... the length of the arrays X and Y
# REAL(8), INTENT(in) :: t
# INTEGER, INTENT(in) :: n
# REAL(8), INTENT(in) :: X(n), Y(n)
# REAL(8), INTENT(out) :: val
# REAL(8) :: f, denum
# INTEGER :: i, j
function lagrange_interp(X, Y, xx)
    N = size(X,1)
    retval = 0.0
    for j in 1:N
        f = 1.0
        den = 1.0
        for i in 1:N
            if i == j
                continue
            end
            f = f * ( xx - X[i] )
            den = den * ( X[j] - X[i] )
        end
        retval = retval + Y[j] * f/den
    end
    return retval
end


function matrix_construct(N::Int64, q::Complex{Float64}, alphas::Array{Float64, 1}; flag::String="False", bc::String = "Neumann")
    l = ones(N);
    if bc == "Neumann"
        d = [4r^2 for r in 0:N]; # N+1 array
        if flag == "True"  # This is Mathieu's canonical form. Or if first element is non-zero
            l[1] = sqrt(2) * l[1];
        end
    elseif bc == "Dirichlet"
        d = [4r^2 for r in 1:N+1]; # N+1 array
    else
        error("BC not recognized. Try `Neumann` or `Dirichlet`")
    end
    A = spdiagm(0 => d);
    for i in 0:length(alphas[2:end])
        if alphas[i+1] != 0
            as = alphas[i+1]*l[1:N-i];
            A = A + spdiagm((i+1) => q.*as, -(i+1)=> q.*as);
        end
    end
    return A
end

function eigval_iterator(N::Int64, q::Array{Complex{Float64}, 1}, alphas::Array{Float64, 1}; flag::String="False", bc::String="Neumann")
    if q[1] == 0
        q[1] = 0.5*(q[1]+q[2]);
    end
    A = matrix_construct(N, q[1], alphas; flag, bc);
    l, = eigs(A, nev= N-1, which=:SM);
    l = round.(l, digits=2);
    I = sortperm(l, lt = (x,y) -> real(x)==real(y) ? imag(x)<imag(y) : real(x)<real(y));
    a = l[I];
    for qf in q[2:end]
        A = matrix_construct(N, qf, alphas; flag, bc);
        l, = eigs(A, nev=N-1, which=:SM);
        l = round.(l, digits=2);
        I = sortperm(l, lt = (x,y) -> real(x)==real(y) ? imag(x)<imag(y) : real(x)<real(y));
        a = hcat(a, l[I]);
    end
    if bc == "Neumann"
        var = "a"  # eigenvalue a_2n
        m = 0  # sets lowest index in a
    elseif bc == "Dirichlet"
        var = "b"  # eigenvalue b_2n+2
        m = 2  # sets lowest index in b
    end
    vals = Dict(var*string(m) => a[1, :], "q"=> q);  # initialized Dict
    for n in 1:length(l)-1
        vals[var*string(2n+m)] = a[n+1, :];
    end
    return vals
end


using MultipleViewGeometry, MultipleViewGeometry.ModuleMove, LinearAlgebra, Test

# Intrinsic and extrinsic parameters for the first camera.
𝐊₁ = zeros(3,3)
𝐊₁[1,1] = 10
𝐊₁[2,2] = 10
𝐊₁[3,3] = 1
𝐑₁ = rotxyz(deg2rad(10), deg2rad(15), deg2rad(45))
𝐭₁ = [-250.0, 0.0, 2500.0]

# Intrinsic and extrinsic parameters for the second camera.
𝐊₂ = zeros(3,3)
𝐊₂[1,1] = 5
𝐊₂[2,2] = 5
𝐊₂[3,3] = 1
𝐑₂ =  rotxyz(deg2rad(10), deg2rad(15), deg2rad(45))
𝐭₂ = [250.0,   0.0, 2500.0]

𝐅 = construct(FundamentalMatrix(),𝐊₁,𝐑₁,𝐭₁,𝐊₂,𝐑₂,𝐭₂)
𝐄 = construct(EssentialMatrix(),𝐅, 𝐊₁, 𝐊₂)

# Result 9.17 of R. Hartley and A. Zisserman, “Two-View Geometry,” Multiple View Geometry in Computer Vision
# A 3 by 3 matrix is an essential matrix if and only if two of its singular values
# are equal, and the third is zero.
U, S , V = svd(𝐄)

@test isapprox.(S[1], S[2]; atol = 1e-14)
@test isapprox.(S[3], 0.0; atol = 1e-10)


using DifferentialEquations
using LinearAlgebra
using Plots
using ForwardDiff
using ProgressMeter

# Thermal Model

function generate1DStencil!(A, N, Δx)
    # Generate the centered difference matrix manually
    for i in 1:N, j in 1:N

    if abs(i-j)<=1
        (A[i,j]+=1)
    end
    if i==j
        (A[i,j]-=3)
    end

    end

    A = A/(Δx^2)
end

function double_partial(x)
    dx = zeros(size(x))
    for i in 2:length(dx)-1
      dx[i] = -1*x[i-1] + 2*x[i] + -1*x[i+1]
    end
    dx[1] = 2*x[1] + -1*x[2]
    dx[end] = -1*x[end-1] + 2*x[end]
    dx
end

# Discretize in time and space
Δx = 1
x = Δx:Δx:100-Δx # Solve only for the interior: the endpoints are known to be zero!

Δt = 0.1
t = Δt:Δt:1000-Δt # Solve only for the interior: the endpoints are known to be zero!

N = length(x)
A = zeros(N,N)
generate1DStencil!(A, N, Δx)

# f(u, p, t) = zeros(size(u))
f(u, p, t) = gen_forcing(p)
# u0_func(x) = sin.(2π * x)

# Given x coord of components, give initial conditions that are equiv to a gaussian around that coordinate
function gen_forcing(positions)
    out_vec = zeros(size(x))
    out_vec = convert.(eltype(positions),out_vec)
    for pos in positions
        out_vec += gaussian.(x, pos, 5.0)
        # out_vec += box.(x, pos)
    end
    out_vec
end

function gaussian(x, mu, sig)
    return exp(-(x - mu)^2.0 / (2 * sig^2.0))
end

function box(x, pos)
    sz = 5.0
    if pos-sz <= x <= pos+sz
        return 1.0
    else
        return 0.0
    end
end

function heat_eq(u, p, t)
    du = A * u + f(u,p,t)
    du
end

# Predict
function predict(positions)
    # Simply roll this forward in time and we get our solution
    tspan = (0.0, 1.0)
    # tspan_dual = convert.(eltype(positions),tspan)
    # @show typeof(positions)
    # u₀ = gen_forcing(positions)
    u₀ = zeros(size(x))
    u_dual = convert.(eltype(positions),u₀)
    # @show typeof(u₀)
    prob = ODEProblem(heat_eq, u_dual, tspan, positions)
    sol = solve(prob)
    # @show typeof(sol[end])

    # display("Finished ODE Solve")
    sol
end

# Loss: Given a list of positions, what is the cost / loss of the
function loss(positions)
    simulation = predict(positions)
    # display(simulation)
    final_state = simulation[end]
    # display(size(final_state)[1])
    # display(final_state)
    return norm(final_state, 2)
end

# Gradient Descent: Optimize initial positions
function optimize(positions, η=0.1; plt=false)

    @showprogress for idx in 1:1000
        grads = ForwardDiff.gradient(s -> loss(s), positions) # Magic
        positions .-= η*grads
        # display(positions)
        # display(loss(positions))

        if loss(positions) < 0.1
            break
        end

        if plt
            sol = predict(positions)
            plt_sol(sol)
        end
    end

    positions
end

function plot_loss_surf()
    loss_surf = zeros((90, 90))
    for x_1 in 5.0:94.0
        for x_2 in 5.0:94.0
            loss_surf[Int(x_1) - 4, Int(x_2) - 4] = loss([x_1, x_2])
        end
    end

    # sol_f(x, y) = loss_surf[x, y]
    plot(loss_surf, st=:heatmap, color=:viridis)

end

# optimize([25.0, 30.0])

function plt_sol(sol)
    sol_f(t, x) = sol[t, x]
    display(plot(1:length(sol[end]),
            1:length(sol.t),
            sol_f,
            st=:surface,
            color=:viridis,
            xlabel="Position",
            ylabel="Time (sec)",
            title="1D Temperature Over Time"))
end

function plt_1D(sol)
    display(plot(sol[end]))
end

function plt_1D_w_pos(sol, positions)
    display(plot(sol[end],
                 linewidth=5,
                 xlabel="Position (meters)",
                 ylabel="Temperature",
                 label="Component positions at " * first(string(positions[1]),4) * " and " * first(string(positions[2]),4)))
end

function optimize_and_save_thermal(positions, η=0.1)
    solns = []
    append!(solns, [deepcopy(positions)])
    @showprogress for idx in 1:1000
        grads = ForwardDiff.gradient(s -> loss(s), positions) # Magic
        positions .-= η*grads
        # display(positions)
        # display(loss(positions))

        if loss(positions) < 0.1
            break
        end

        if idx in [500,1000]
            append!(solns, [deepcopy(positions)])
        end
    end

    solns
end

function test_and_save_thermal()
    p = [25.0, 30.0]
    solns = optimize_and_save_thermal(p)
    display(plot())
    i = 1
    idxs = [1,500,1000]
    for pos in solns
        sol = predict(pos)
        plt_1D_w_pos(sol, pos)
        plot!(title="1D Thermal Optimizer: Iteration " * string(idxs[i]), ylims = (0.0,2.0))
        savefig("plots/thermal_plot_"*string(i))
        i += 1
    end
end


# least squares penalty

"""
  LeastSquares(A, b, λ=1.0)

Returns the function `f(x) = (λ/2)⋅||Ax-b||^2`.
"""

type LeastSquares{R <: RealOrComplex} <: ProximableFunction
  A::AbstractArray{R,2}
  b::AbstractArray{R,1}
  Atb::AbstractArray{R,1}
  lambda::Real
  gamma::Real
  S::AbstractArray{R,2}
  U::Union{LinAlg.Cholesky, SparseArrays.CHOLMOD.Factor}
  function LeastSquares(A::AbstractArray{R,2}, b::AbstractArray{R,1}, lambda::Real)
    if size(A, 1) != length(b)
      error("A and b have incompatible dimensions")
    end
    if lambda <= 0
      error("lambda must be positive")
    end
    if size(A,1) >= size(A,2)
      new(A, b, A'*b, lambda, -1, A'*A)
    else
      new(A, b, A'*b, lambda, -1, A*A')
    end
  end
end

LeastSquares{R <: RealOrComplex}(A::AbstractArray{R,2}, b::AbstractArray{R,1}, lambda::Real=1.0) =
  LeastSquares{R}(A, b, lambda)

function (f::LeastSquares{R}){R <: RealOrComplex}(x::AbstractArray{R,1})
  return (f.lambda/2)*vecnorm(f.A*x - f.b, 2)^2
end

function factor_step!{R <: RealOrComplex}(f::LeastSquares{R}, gamma::Real)
  # factor step, two cases: (1) tall A, (2) fat A
  lamgam = f.lambda*gamma
  if issparse(f.A)
    f.U = cholfact(f.S; shift=1.0/lamgam)
  else
    f.U = cholfact(f.S + I/lamgam)
  end
  f.gamma = gamma
end

function prox!{R <: RealOrComplex}(f::LeastSquares{R}, x::AbstractArray{R,1}, y::AbstractArray{R,1}, gamma::Real=1.0)
  # if gamma different from f.gamma then call factor_step!
  if gamma != f.gamma
    factor_step!(f, gamma)
  end
  lamgam = f.lambda*gamma
  # solve step, two cases: (1) tall A, (2) fat A
  q = f.Atb + x/lamgam
  if size(f.A,1) >= size(f.A,2)
    y[:] = f.U\q
  else
    y[:] = lamgam*(q - (f.A'*(f.U\(f.A*q))))
  end
  return (f.lambda/2)*norm(f.A*y-f.b, 2)^2
end

fun_name(f::LeastSquares) = "least-squares penalty"
fun_dom{R <: Real}(f::LeastSquares{R}) = "AbstractArray{Real,1}"
fun_dom{C <: Complex}(f::LeastSquares{C}) = "AbstractArray{Complex,1}"
fun_expr(f::LeastSquares) = "x ↦ (λ/2)||A*x - b||^2"
fun_params(f::LeastSquares) = string("λ = $(f.lambda), A = ", typeof(f.A), " of size ", size(f.A), ", b = ", typeof(f.b), " of size ", size(f.b))

function prox_naive{R <: RealOrComplex}(f::LeastSquares, x::AbstractArray{R,1}, gamma::Real=1.0)
  lamgam = f.lambda*gamma
  y = (f.A'*f.A + I/lamgam)\(f.Atb + x/lamgam)
  fy = (f.lambda/2)*norm(f.A*y-f.b)^2
  return y, fy
end


using Plots, LaTeXStrings, Statistics, JLD, BenchmarkTools, StatsBase, SpecialFunctions, ProgressMeter

Integral_answer=sqrt(π)/2 * erf(2)

function SSMC(f :: Function, a :: Real, b :: Real, N :: Integer )
    f₍ₓ₎= f.((b-a).*rand(N) .+ a)
    ∫f₍ₓ₎dx= (b-a)*(mean(f₍ₓ₎))
    error=std(f₍ₓ₎)/sqrt(N)
    return ∫f₍ₓ₎dx, error
end

function ISMC(f :: Function, g :: Function, y :: Function, a :: Real, b :: Real, N :: Integer)
    drivation= x-> f(x)/g(x)
    samples= drivation.(y.(rand(N)))
    ∫g₍ₓ₎dx= 1-ℯ^(-2)
    ∫f₍ₓ₎dx= ∫g₍ₓ₎dx * mean(samples)
    error= ∫g₍ₓ₎dx * std(samples)/sqrt(N)
    return ∫f₍ₓ₎dx, error
end

nList=Integer.([10^n for n in 1:8])
SSMC_res=[]
prog= Progress(length(nList))
n=0
for n in nList
    I, error=SSMC(x-> ℯ^(-x^2), 0, 2, n)
    bench = @benchmark SSMC(x->ℯ^(-x^2), 0, 2, n)
    push!(SSMC_res, (I, error, mean(bench).time, bench))
    next!(prog)
end
ISMC_res=[]
n=0
prog= Progress(length(nList))
for n in nList
    I, error=ISMC(x->ℯ^(-x^2), x -> ℯ^(-x), x -> -log(1-(1-ℯ^(-2))*x), 0, 2, n)
    bench = @benchmark ISMC(x->ℯ^(-x^2), x -> ℯ^(-x), x -> -log(1-(1-ℯ^(-2))*x), 0, 2, n)
    push!(ISMC_res, (I, error, mean(bench).time, bench))
    next!(prog)
end
save("../../chapter7/7.1/data.jld","ISMC", ISMC_res, "SSMC", SSMC_res, "samples", nList)
AnswerList=[Integral_answer for i in 1: length(nList)]
scatter(log.(nList), [getindex.(SSMC_res, 1) getindex.(ISMC_res, 1) AnswerList], markercolor=[:blue :purple :orange], label=nothing)
plot!(log.(nList), [getindex.(SSMC_res, 1) getindex.(ISMC_res, 1) AnswerList], linecolor=[:blue :purple :orange],
    dpi=600,labels=["SSMC" "ISMC" "Absolute Result"], ylabel="Integral", xlabel="Number of Samples",
        title=L"Monte\ Carlo\ Results\ for\ I= \int{}_{0}^{2}\ e^{-x^2}\ dx" )
savefig("C:\\Users\\Narges\\Documents\\GitHub\\computational_physics\\chapter7\\7.1\\Figs\\ResultsPlot.png")
scatter(log.(nList), [getindex.(SSMC_res, 2) getindex.(ISMC_res, 2)], markercolor=[:blue :purple], label= nothing)
plot!(log.(nList), [getindex.(SSMC_res, 2) getindex.(ISMC_res, 2)],
     labels=["SSMC" "ISMC"], xlabel="Number of Samples", ylabel=" Error", dpi=600, linecolor=[:blue :purple],
        title=L"Monte\ Carlo\ Errors\ of\ the\ Results\ for\ I= \int{}_{0}^{2}\ e^{-x^2}\ dx")
savefig("C:\\Users\\Narges\\Documents\\GitHub\\computational_physics\\chapter7\\7.1\\Figs\\ErrorPlot.png")


using Yao
using QuDiffEq
using LinearAlgebra
using BitBasis
using OrdinaryDiffEq
using Plots
using Test
# vertex and egde values
vertx = 7
ege = 8

#Constructing incidence matrix B
B = zeros(vertx,ege)
@inbounds for i in 1:vertx
    B[i,i] = -1
    B[i,i+1] = 1
end
B[1,1] = 1

de = 0.5
sn = sin.((0.0:de:(vertx-1)*de)*2*pi/((vertx-1)*de))
u0 = ComplexF64.(sn)

# Intial conditions (stationary to begin with)
u1 = [u0; zero(u0); 0.0; 0.0]
u_ = Float64[u0 zero(u0)]

k = 2   # order in Taylor expansion
t = 1e-2 # time step
B_t = transpose(B)
n = 11  # number of steps
a = 1e-1 # spactial discretization

function make_hamiltonian(B,B_t,a)
    vertx,ege = size(B)
    n = nextpow(2,vertx+ege)
    H = zeros(n,n)
    @inbounds H[1:vertx,vertx+1:vertx+ege] = B
    @inbounds H[vertx+1:vertx+ege,1:vertx] = B_t
    H = -im/a*H
    return H
end

function do_pde(ϕ,B,B_t,k,t,a,n)
    vertx, = size(B)
    H = make_hamiltonian(B,B_t,a)
    res = Array{Array{ComplexF64,1},1}(undef,n)
    res[1] = @view ϕ[1:vertx]
    for i in 2:n
    r, N = taylorsolve(H,ϕ,k,t)
    ϕ = N*vec(state(r))
    res[i] = @view ϕ[1:vertx]
    end
    res_real = real(res)
    return res_real
end

#Dirichlet
res1 = do_pde(u1,B,B_t,k,t,a,n)

const D1 = -1/(a*a)*B*B_t # Laplacian Operator
function f(du,u,p,t)
    buffer, D = p
    u1 = @view(u[:,1])
    u2 = @view(u[:,2])
    mul!(buffer, D, u2)
    Du = buffer

    du[:,1] = Du
    du[:,2] = u1
end

tspan = (0.0,0.1)
prob1 = ODEProblem(f,u_,tspan,(zero(u_[:,1]), D1))
sol1 = solve(prob1,Tsit5(),dt=0.01,adaptive = false)
s1 = Array{Array{Float64,1},1}(undef,n)
for i in 1:n
    s1[i] = @view (sol1.u[i][:,1])
end

@test isapprox.(res1,s1, atol = 1e-2) |> all


mutable struct MultivariateGaussianDist
    _mu::AbstractArray{Float64,1}
    _cov::AbstractArray{Float64,2}
    function MultivariateGaussianDist(dim::Int64)
        new(zeros(dim), Matrix{Float64}(I, dim, dim))
    end
end

function pdf(gaussian::MultivariateGaussianDist, x::AbstractArray{Float64,1})
    @assert size(x)[1] == size(gaussian._mu)[1]
    N = size(x)[1]
    d = x - gaussian._mu
    quadratics = transpose(d) * gaussian._cov * d
    @assert typeof(quadratics) == Float64
    return exp(-0.5 * quadratics) / sqrt(det(gaussian._cov)) / (2 * pi)^(0.5 * N)
end

function pdf(gaussian::MultivariateGaussianDist, X::AbstractArray{Float64,2})
    # X consists of list of samples [x_1, x_2, ,,, x_(n_samples)], and x_i is a vertical vector
    @assert size(X)[1] == size(gaussian._mu)[1]
    N = size(X)[1]
    n_samples = size(X)[2]
    # In Julia lang, 1-D AbstractArray is vertical vector
    d = X .- gaussian._mu
    #quadratics = diag(transpose(d) * gaussian._cov * d);
    quadratics = [transpose(d[:, i]) * gaussian._cov * d[:, i] for i = 1:n_samples]
    @assert ndims(quadratics) == 1 && size(quadratics)[1] == n_samples
    return exp.(-0.5 * quadratics) / sqrt(det(gaussian._cov)) / (2 * pi)^(0.5 * N)
end

function fitting(gaussian::MultivariateGaussianDist, X::AbstractArray{Float64,1})
    gaussian._mu = X
    gaussian._cov = cov(X, corrected = true)
end

function fitting(gaussian::MultivariateGaussianDist, X::AbstractArray{Float64,2})
    # X consists of the list of samples [x_1, x_2, ,,, x_(n_samples)]
    @assert size(X)[1] == size(gaussian._mu)[1]
    n_samples = size(X)[2]
    N = size(X)[1]
    gaussian._mu = collect(reshape(mean(X, dims = 2), N))
    gaussian._cov = cov(X, dims = 2, corrected = true)
end


function AmericanSpreadCallBinomial(K, T, S1, S2, sig1, sig2, div1, div2, rho, r, N)

  dt = T/N ## Time to Maturity/ Number of Steps
  nu1 = r - div1 - 0.5 * sig1^2
  nu2 = r - div2 - 0.5 * sig2^2

  dx1 = sig1 * sqrt(dt)
  dx2 = sig2 * sqrt(dt)

  disc = exp(-r * dt)

  puu = (dx1 * dx2 + (dx2 * nu1 + dx1 * nu2 + rho * sig1 * sig2) * dt) / (4 * dx1 * dx2) * disc

  pud = (dx1 * dx2 + (dx2 * nu1 - dx1 * nu2 - rho * sig1 * sig2) * dt) / (4 * dx1 * dx2) * disc

  pdu = (dx1 * dx2 + (-dx2 * nu1 + dx1 * nu2 - rho * sig1 * sig2) * dt) / (4 * dx1 * dx2) * disc

  pdd = (dx1 * dx2 + (-dx2 * nu1 - dx1 * nu2 + rho * sig1 * sig2) * dt) / (4 * dx1 * dx2) * disc

  edx1 = exp(dx1)
  edx2 = exp(dx2)

  S1t = linspace(-N, N, 2N+1)
  S2t = linspace(-N, N, 2N+1)

  S1t[1] = S1 * exp(-N * dx1)
  S2t[1] = S2 * exp(-N * dx2)

  for j in 2:length(-N:N)
    S1t[j] = S1t[j-1] * edx1
    S2t[j] = S2t[j-1] * edx2
  end

  C = Array(Float64, (2N+1,2N+1))

  for j = 1:2:length(-N:N)
    for k = 1:2:length(-N:N)
      C[k, j] = max(0.0, S1t[j] - S2t[k] - K)
    end
  end

  for i in range(N-1, -1, N)
    for j in range(-i, 2, i+1) # potentially this is i^2
      for k in range(-i, 2, i+1) # potentiall this is i^2 as well
        C[k+(N+1), j+(N+1)] = pdd * C[k+(N+1)-1, j+(N+1)-1] + pud * C[k+(N+1)-1, j+(N+1)+1] + pdu * C[k+(N+1)+1, j+(N+1)-1] + puu * C[k+(N+1)+1, j+(N+1)+1]
        C[k+(N+1), j+(N+1)] = max(C[k+(N+1), j+(N+1)], S1t[j+(N+1)] - S2t[k+(N+1)] - K)
      end
    end
  end

  C[N+1, N+1]

end


using StatsBase, Distributions, Plots

function rouletteSpins(r,p)
    x = 0
    wins = 0
    while true
        x += 1
        if rand() < p
            wins += 1
            if wins == r
                return x
            end
        end
    end
end

r, p, N = 5, 18/37,10^6
xGrid = r:r+15

mcEstimate = counts([rouletteSpins(r,p) for _ in 1:N],xGrid)/N

nbDist = NegativeBinomial(r,p)
nbPmf = [pdf(nbDist,x-r) for x in xGrid]

plot( xGrid, mcEstimate,
	line=:stem, marker=:circle, c=:blue,
	ms=10, msw=0, lw=4, label="MC estimate")
plot!( xGrid, nbPmf, line=:stem,
	 marker=:xcross, c=:red, ms=6, msw=0, lw=2, label="PMF",
	 xlims=(0,maximum(xGrid)), ylims=(0,0.2),
	 xlabel="x", ylabel="Probability")


using LinearAlgebra
using ForwardDiff
using Plots
using Distributions

# Bundled Gradients through Contact via Randomized Smoothing

function gd(x0; iters=1000, verbose=true, visualize=true)
    x = copy(x0)
    α = 0.001

    for i = 1:iters
        grad = ∇f(x)
        x -= α * grad
    end

    if verbose
        println("grad. norm: ", norm(∇f(x)))
    end

    if visualize
        x_range = range(-1.0, stop = 1.0, length = 1000)
        plt = plot(x_range, f.(x_range), label="", color=:black, width=2.0)
        plt = scatter!([x0,], [f(x0),], color=:red, label="")
        plt = scatter!([x,], [f(x),], color=:green, label="")
        display(plt)
    end

    return x
end

function gd(x0; iters=1000, verbose=true, visualize=true,
    mode=:gradient,
    μ=0.0, σ=[0.001], σ_schedule=1000, N=100,
    xmin=-1.0, xmax=1.0)
    x = copy(x0)
    α = 0.001

    for s in σ
        dist = Normal(μ, s)
        for i = 1:iters
            if mode == :gradient
                grad = ∇f(x)
            elseif mode == :smooth_gradient
                grad = sum([∇f(x + rand(dist, 1)[1]) for j = 1:N]) / N
            elseif mode == :zero_order_gradient
                fj = []
                for j = 1:N
                    ϵ = rand(dist, 1)[1]
                    # push!(fj, (f(x + ϵ) - f(x)) / (1.0 * ϵ))
                    push!(fj, (f(x + ϵ) - f(x - ϵ)) / (2.0 * ϵ))
                    grad = sum(fj) / N
                end
            else
                nothing
            end
            x -= α * grad
        end
    end

    if verbose
        println("grad. norm: ", norm(∇f(x)))
    end

    if visualize
        x_range = range(xmin, stop = xmax, length = 1000)
        plt = plot(x_range, f.(x_range), label="", color=:black, width=2.0)
        for s in σ
            dist = Normal(μ, s)
            if mode != :gradient
                plt = plot!(x_range, [sum([f(xt + rand(dist, 1)[1]) for j = 1:1000]) / 1000  for xt in x_range],
                    label="σ=$s", width=1.0, legend=:top)
            end
        end
        plt = scatter!([x0,], [f(x0),], color=:red, label="")
        plt = scatter!([x,], [f(x),], color=:green, label="")
        display(plt)
    end

    return x
end

# example 1
f(x) = x^2.0 + 0.1 * sin(20.0 * x)
∇f(x) = 2.0 * x + 2.0 * cos(20.0 * x)

# initial point
x0 = -0.75

# gradient descent
x_sol = gd(x0, iters=1000,
           σ=[0.5, 0.25, 0.1, 0.01, 0.001],
           mode=:smooth_gradient)

# example 2
function f(x)
    if x >= 0.0
        return 1.0
    else
        return 0.0
    end
end
∇f(x) = 0.0

# initial point
x0 = 1.0

# gradient descent
x_sol = gd(x0, iters=10000,
           σ=[1.0, 0.5, 0.1],#, 0.25, 0.1, 0.01, 0.001],#, 0.075, 0.05, 0.01],
           N = 100,
           mode=:zero_order_gradient,
           xmin=-2.0, xmax=2.0)

# example 3
function f(x)
    if x >= 0.0
        return -1.0 + x
    else
        return 1.0 - x
    end
end
function ∇f(x)
    if x >= 0.0
        return 1.0
    else
        return -1.0
    end
end

x0 = 2.0
x_sol = gd(x0, iters=1000,
           σ=[1.0, 0.5, 0.1],#, 0.25, 0.1, 0.01, 0.001],#, 0.075, 0.05, 0.01],
           N = 100,
           mode=:zero_order_gradient,
           xmin=-2.0, xmax=2.0)


using Pkg
Pkg.activate(pwd())

# # Plots.jl

using Plots

x = range(0, 2π; length = 100)
y = sin.(x)

plot(x, y)

#+

y = hcat(sin.(x), cos.(x))

plot(x, y);
plot!(x, sin.(x .+ π/4))

#+

plt = plot(x, hcat(sin.(x), cos.(x)));
plot!(plt, x, sin.(x .+ π/4))

# ## Plot attributes

x = range(0, 2π; length = 100)
y = hcat(sin.(x), cos.(x))

plot(x, y;
    label = ["sine" "cosine"],
    xguide = "x",
    yguide = "y",
    legend = :bottomleft,
    title = "Trigonometric functions",
    xticks = (0:0.5π:2π, ["0", "0.5π", "π", "1.5π", "2π"]),
    color = [:red :blue],
    linestyle = [:dash :dot],
    linewidth = [2 4],
)

#+

n = 200
x = range(0, 2π; length = n)
linewidth = range(1, 50; length = n)
color = palette(:viridis, n)
xlims = (0, 7)
ylims = (-1.2, 1.2)
label = ""

plot(x, sin.(x); linewidth, color, label, xlims, ylims)

#+

plot(x, [sin.(x) cos.(x)]; linewidth, color = [color :red], label, xlims, ylims)

#+

plotattr(:Series)

#+

plotattr("title")

#+

plotattr("xguide")

#+

plotattr("guide")

# ### Exercise:
# Consider the following set of equations
#
# $$
# \begin{aligned}
# x(t) & = \cos(3t), \\
# y(t) & = \sin(2t),\\
# \end{aligned}
# $$
#
# where $t \in [0, 2\pi]$. Create a plot of the curve described by the equations above.
# Use plot attributes to set the following properties:
#
# 1. The line width should start at `1`, increase to `50` and then decrease back to `1`.
# 2. The line color should change with the changing line width.
#
# Use `:viridis` color scheme or any other
# [color scheme](http://docs.juliaplots.org/latest/generated/colorschemes/) supported by
# the Plots package. Use additional plot attributes to get a nice looking graph.
#
# **Hints:**
# - use the `pallete` function combined with the `collect` function to generate a vector of
# colors from the `:viridis` color scheme.
# - remove all decorators by using: `axis = nothing`, `border = :none`.
#
# ---
# ### Solution:

n = 1000
t = range(0, 2π; length = n)

fx(t) = cos(3t)
fy(t) = sin(2t)

#+

linewidth = vcat(
    range(1, 50; length = n ÷ 2),
    range(50, 1; length = n - n ÷ 2)
)

#+

c = palette(:viridis, 2);
typeof(c)

#+

c = collect(palette(:viridis, 2))

#+

color = vcat(
    collect(palette(:viridis, n ÷ 2)),
    collect(palette(:viridis, n - n ÷ 2; rev = true))
)

#+

plot(fx.(t), fy.(t);
    linewidth,
    color,
    lims = (-1.2, 1.2),
    legend = false,
    axis = nothing,
    border = :none,
)

# ---
#
# ## Function plotting

t = range(0, 2π; length = 100)

plot(t, [sin, cos]; label = ["sine" "cosine"])

#+

plot(sin, x -> sin(2x), t; linewidth = 2, label = "")

#+

plot(sin, x -> sin(2x), 0, 2π, 100; linewidth = 2, label = "")

# ### Exercise:
# Create a plot given by the following set of equations:
#
# $$
# \begin{aligned}
# x(t) & = (a + b)\cos(t) - b \cdot \cos \left( \left(\frac{a}{b} + 1 \right)t \right), \\
# y(t) & = (a + b)\sin(t) - b \cdot \sin \left( \left(\frac{a}{b} + 1 \right)t \right), \\
# \end{aligned}
# $$
# where $a = 4.23$, $b = 2.35$ and $t \in [-15, 20]$. Use additional plot attributes
# to get a nicely looking graph.
#
# ---
# ### Solution:

fx(t; a = 4.23, b = 2.35) = (a + b)*cos(t) - b*cos((a/b + 1)*t)
fy(t; a = 4.23, b = 2.35) = (a + b)*sin(t) - b*sin((a/b + 1)*t)

plot(fx, fy, -15, 20, 500;
    linewidth = 2,
    legend = false,
    axis = nothing,
    border = :none,
)

# ---
#
# Changing the plotting series

x = range(0, 2π; length = 100)
y = sin.(x)

plot(x, y; seriestype = :scatter)

#+

scatter(x, y)

# ### Exercise:
# Consider the following function:
#
# $$
# f(x, y) = \frac{x^2 \cdot y^2}{x^4 + y^4}.
# $$
#
# Draw this function for $x, y \in [-5, 5]$. Use the following three plot series
# `contourf`, `heatmap`, and `surface` with the following settings:
# - `:viridis` color scheme,
# - camera angle `(25, 65)`,
# - no legend, color bar, or decorators (`axis`, `frame` and `ticks`).
#
# ---
# ### Solution:

x = range(-5, 5; length = 200)

fz(x, y) = x^2*y^2/(x^4 + y^4)

#+

kwargs = (
    color = :viridis,
    legend = false,
    cbar = false,
    axis = nothing,
    border = :none,
)

#+

contourf(x, x, fz; kwargs...) # or plot(x, x, fz; seriestype = :contourf, kwargs...)
heatmap(x, x, fz; kwargs...)
surface(x, x, fz; camera = (25, 65), kwargs...)

# ---
#
# Subplots

x = range(0, 2π; length = 100)

plot(x, [sin, cos, tan, sinc];
    layout = 4,
    linewidth = 2,
    legend = false,
    title = ["1" "2" "3" "4"],
)

#+

plot(x, [sin, cos, tan, sinc];
    layout = grid(4, 1; heights = [0.1 ,0.4, 0.4, 0.1]),
    linewidth = 2,
    legend = false,
)

#+

l = @layout [a ; b{0.3w} c]
plot(x, [sin, cos, tan]; layout = l, linewidth = 2, legend = false)

#+

linewidth = range(1, 20; length = 100)

p1 = plot(x, sin; legend = false, line_z = 1:100, color = :viridis, linewidth);
p2 = plot(x, cos; legend = false, line_z = 1:100, color = :Blues_9, linewidth);
p3 = plot(x, tan; legend = false, line_z = 1:100, color = :hsv, linewidth);

l = @layout [a ; b{0.3w} c]
plot(p1, p2, p3; layout = l)

# ## Animations

n = 300

plt = plot(Float64[], [sin, cos];
    legend = false,
    xlims = (0, 6π),
    ylims = (-1.1, 1.1),
    linewidth = range(1, 20; length = n),
    color = palette(:viridis, n),
    axis = nothing,
    border = :none
)

anim = Animation()

for x in range(0, 6π; length = n)
    push!(plt, x, [sin(x), cos(x)])
    frame(anim)
end

gif(anim, joinpath(pwd(), "lecture_05", "animsincos.gif"), fps = 15)

#+

x = range(-5, 5; length = 400)
fz(x, y) = x^2*y^2/(x^4 + y^4)

plt = surface(x, x, fz;
    camera = (30, 65),
    color = :viridis,
    legend = false,
    axis = nothing,
    border = :none,
    cbar = false,
)

anim = @animate for i in vcat(30:60, 60:-1:30)
    plot!(plt, camera = (i, 65))
end

gif(anim, joinpath(pwd(), "lecture_05", "animsurf.gif"), fps = 15)


using DistanceAndSimilarity
using Test

@testset "Function test" begin

    @test euclidianDistance([1,1],[1,3]) == 2
    @test manhattanDistance([3,45,7,2],[2,54,13,15]) == 29
    @test cosineDistance([3,45,7,2],[2,54,13,15]) == 0.02771574828765011
    @test minkowskiDistance([1,2,3],[1,2,3],2) == 0
    @test jaccardDistance([1,2,3],[3,4,5]) == (1- 1/5)
    @test jaccardSimilarity([1,2,3],[3,4,5]) == 1/5

    @test errorCheck([0,0],[1,1,1],1) == -1
    @test errorCheck([1,1],[1,1],0) == -1

    end


function subbac!(u::Matrix{Float64}, b::Vector{Float64})
# Back-substitution on an Upper Triangle
  n = size(u, 1)
  for i in n:-1:1
    total = b[i]
    if i < n
      for j in i+1:n
        total -= u[i, j] * b[j]
      end
    end
    b[i] = total / u[i, i]
  end
end


function proj_ortogonal(v,w)

    z = ( (v'*w)/(w'*w) ) * w;

    return z;

end


function countingsort!(v::Vector{<:Integer})
    mini, maxi = extrema(v)

    keys = zeros(Int, maxi-mini+1)

    for key in v
        keys[key-mini+1] += 1
    end

    i = 1
    for (key, amount) in enumerate(keys)
        v[i:i+amount-1] .= key+mini-1
        i += amount
    end
    return v
end

v = [144, 89, 4, 9, 95, 12, 86, 25]
println("unsorted array = ",v)

countingsort!(v)

println("counting sorted array = ", v)


using LinearAlgebra
using SparseArrays
using Plots

"This code solves the 2D Laplace's equation with nonzero Dirichlet and Neumann
boundary conditions using a finite difference method."

m = 50 # number of points
x = LinRange(-1,1,m+2)
y = LinRange(-1,1,m+2)
xint = x[2:end-1]
yint = y[2:end-1]
h = x[2]-x[1]

T = 2.0 # final time
dt = .01 # timestep

id(i,j,m) = i + (j-1)*m # return global index from i,j
function FD_matrix_2D(h,m)
    A = spzeros(m*m,m*m)
    for i = 1:m, j = 1:m # loop thru x,y indices
        A[id(i,j,m),id(i,j,m)] = 4/h^2
        if i > 1 # avoids leftmost line of nodes
            A[id(i,j,m),id(i-1,j,m)] = -1/h^2 # x-derivative
        end
        if i < m # avoids rightmost line of nodes
            A[id(i,j,m),id(i+1,j,m)] = -1/h^2
        end
        if j > 1
            A[id(i,j,m),id(i,j-1,m)] = -1/h^2 # y-derivative
        end
        if j < m
            A[id(i,j,m),id(i,j+1,m)] = -1/h^2
        end
    end
    return A
end

A = FD_matrix_2D(h,m)

# initial condition
u0(x,y) = exp(-100*(x^2+y^2))
u0(x,y) = 0.0

# forcing
f(x,y,t) = exp(-25*((x-.25)^2 + (y-.75)^2))*exp(2*t)*(t < 1.5)
# f(x,y,t) = 0.0

F(t) = vec(f.(xint,yint',t)) # zero Dirichlet BCs
u = vec(u0.(xint,yint'))
interval = 1
Nsteps = ceil(Int,T / dt)
dt = T/Nsteps

Bsparse = (I + .5*dt*A)
#B = cholesky(I + .5*dt*A)
B = lu(I + .5*dt*A)

plot()
@gif for i = 1:Nsteps
    t = i*dt
    Favg = .5*(F(t) + F(t+dt))
    u .= B\(u + .5*dt*(Favg-A*u))
    if i%interval==0
        println("on timestep $i / $Nsteps")
        contourf(xint,yint,u,clims=(0,.5)) #vec(reshape(u,m,m)'))
    end
end every interval


# 1D Non-Linear Diffusion tests
# See doi:10.1016/j.camwa.2006.12.077

# TODO: Add more complex tests.

# Packages and inclusions
using ModelingToolkit,DiffEqOperators,LinearAlgebra,Test,OrdinaryDiffEq, DomainSets
using ModelingToolkit: Differential
# Tests
@testset "Test 00: Dt(u(t,x)) ~ Dx(u(t,x)^(-1) * Dx(u(t,x)))" begin
    # Variables, parameters, and derivatives
    @parameters t x
    @variables u(..)
    Dx = Differential(x)
    Dt = Differential(t)
    t_min= 0.
    t_max = 2.
    x_min = 0.
    x_max = 2.
    c = 1.0
    a = 1.0

    # Analytic solution
    analytic_sol_func(t,x) = 2.0 * (c + t) / (a + x)^2

    # Equation
    eq = Dt(u(t,x)) ~ Dx(u(t,x)^(-1) * Dx(u(t,x)))

    # Initial and boundary conditions
    bcs = [u(t_min,x) ~ analytic_sol_func(t_min,x),
           u(t,x_min) ~ analytic_sol_func(t,x_min),
           u(t,x_max) ~ analytic_sol_func(t,x_max)]

    # Space and time domains
    domains = [t ∈ Interval(t_min,t_max),
               x ∈ Interval(x_min,x_max)]

    # PDE system
    @named pdesys = PDESystem([eq],bcs,domains,[t,x],[u(t,x)])

    # Method of lines discretization
    dx = 0.01
    discretization = MOLFiniteDifference([x=>dx],t)
    prob = ModelingToolkit.discretize(pdesys,discretization)

    # Solution of the ODE system
    using OrdinaryDiffEq
    sol = solve(prob,Rosenbrock32())

    # Test against exact solution
    r_space = x_min:dx:x_max
    asf = [analytic_sol_func(t_max,x) for x in r_space]
    Nx = floor(Int64, (x_max - x_min) / dx) + 1
    @variables u[1:Nx](t)
    sol′ = [sol[u[i]][end] for i in 1:Nx]
    @test asf ≈ sol′ atol=0.1

    # Plots
    #using Plots
    #plot(r_space, asf, seriestype = :scatter,label="analytic solution")
    #plot!(r_space, sol′, label="numeric solution")
    #savefig("MOL_NonLinear_Diffusion_1D_Test00.png")

end

@testset "Test 01: Dt(u(t,x)) ~ Dx(u(t,x)^2 * Dx(u(t,x)))" begin

    # Variables, parameters, and derivatives
    @parameters t x
    @variables u(..)
    Dx = Differential(x)
    Dt = Differential(t)
    t_min= 0.
    t_max = 2.
    x_min = 0.
    x_max = 2.
    c = 50.0
    h = 0.50

    # Analytic solution
    analytic_sol_func(t,x) = 0.5 * (x + h) / sqrt(c - t)

    # Equation
    eq = Dt(u(t,x)) ~ Dx(u(t,x)^2 * Dx(u(t,x)))

    # Initial and boundary conditions
    bcs = [u(t_min,x) ~ analytic_sol_func(t_min,x),
           u(t,x_min) ~ analytic_sol_func(t,x_min),
           u(t,x_max) ~ analytic_sol_func(t,x_max)]

    # Space and time domains
    domains = [t ∈ Interval(t_min,t_max),
               x ∈ Interval(x_min,x_max)]

    # PDE system
    @named pdesys = PDESystem([eq],bcs,domains,[t,x],[u(t,x)])

    # Method of lines discretization
    dx = 0.01
    discretization = MOLFiniteDifference([x=>dx],t)
    prob = ModelingToolkit.discretize(pdesys,discretization)

    # Solution of the ODE system
    using OrdinaryDiffEq
    sol = solve(prob,Rosenbrock32())

    # Test against exact solution
    r_space = x_min:dx:x_max
    asf = [analytic_sol_func(t_max,x) for x in r_space]
    Nx = floor(Int64, (x_max - x_min) / dx) + 1
    @variables u[1:Nx](t)
    sol′ = [sol[u[i]][end] for i in 1:Nx]
    @test asf ≈ sol′ atol=0.01

    # Plots
    #using Plots
    #plot(r_space, asf, seriestype = :scatter,label="analytic solution")
    #plot!(r_space, sol′, label="numeric solution")
    #savefig("MOL_NonLinear_Diffusion_1D_Test01.png")

end

@testset "Test 02: Dt(u(t,x)) ~ Dx(1. / (1. + u(t,x)^2) * Dx(u(t,x)))" begin

    # Variables, parameters, and derivatives
    @parameters t x
    @variables u(..)
    Dx = Differential(x)
    Dt = Differential(t)
    t_min= 0.
    t_max = 2.
    x_min = 0.
    x_max = 2.

    # Analytic solution
    analytic_sol_func(t,x) = tan(x)

    # Equation
    eq = Dt(u(t,x)) ~ Dx(1. / (1. + u(t,x)^2) * Dx(u(t,x)))

    # Initial and boundary conditions
    bcs = [u(t_min,x) ~ analytic_sol_func(t_min,x),
           u(t,x_min) ~ analytic_sol_func(t,x_min),
           u(t,x_max) ~ analytic_sol_func(t,x_max)]

    # Space and time domains
    domains = [t ∈ Interval(t_min,t_max),
               x ∈ Interval(x_min,x_max)]

    # PDE system
    @named pdesys = PDESystem([eq],bcs,domains,[t,x],[u(t,x)])

    # Method of lines discretization
    dx = 0.01
    discretization = MOLFiniteDifference([x=>dx],t)
    prob = ModelingToolkit.discretize(pdesys,discretization)

    # Solution of the ODE system
    using OrdinaryDiffEq
    sol = solve(prob,Rosenbrock32()) # TODO: check warnings

    # Test against exact solution
    r_space = x_min:dx:x_max
    asf = [analytic_sol_func(t_max,x) for x in r_space]
    Nx = floor(Int64, (x_max - x_min) / dx) + 1
    @variables u[1:Nx](t)
    sol′ = [sol[u[i]][end] for i in 1:Nx]

    m = max(asf...,sol′...)
    @test asf / m ≈ sol′ / m atol=0.16 # the difference occurs when tan(x) goes to infinite

    # Plots
    #using Plots
    #plot(r_space, asf, seriestype = :scatter,label="Analytic solution")
    #plot!(r_space, sol′, label="Numeric solution")
    #savefig("MOL_NonLinear_Diffusion_1D_Test02.png")

end

@testset "Test 03: Dt(u(t,x)) ~ Dx(1. / (u(t,x)^2 - 1.) * Dx(u(t,x)))" begin

    # Variables, parameters, and derivatives
    @parameters t x
    @variables u(..)
    Dx = Differential(x)
    Dt = Differential(t)
    t_min= 0.
    t_max = 2.
    x_min = 0.
    x_max = 2.

    # Analytic solution
    analytic_sol_func(t,x) = -coth(x)

    # Equation
    eq = Dt(u(t,x)) ~ Dx(1. / (u(t,x)^2 - 1.) * Dx(u(t,x)))

    # Initial and boundary conditions
    bcs = [u(t_min,x) ~ analytic_sol_func(t_min,x),
           u(t,x_min) ~ analytic_sol_func(t,x_min),
           u(t,x_max) ~ analytic_sol_func(t,x_max)]

    # Space and time domains
    domains = [t ∈ Interval(t_min,t_max),
               x ∈ Interval(x_min,x_max)]

    # PDE system
    @named pdesys = PDESystem([eq],bcs,domains,[t,x],[u(t,x)])

    # Method of lines discretization
    dx = 0.01
    discretization = MOLFiniteDifference([x=>dx],t)
    prob = ModelingToolkit.discretize(pdesys,discretization)

    # Solution of the ODE system
    using OrdinaryDiffEq
    sol = solve(prob,Rosenbrock32())

    # Test against exact solution
    r_space = x_min:dx:x_max
    asf = [analytic_sol_func(t_max,x) for x in r_space]
    Nx = floor(Int64, (x_max - x_min) / dx) + 1
    @variables u[1:Nx](t)
    sol′ = [sol[u[i]][end] for i in 1:Nx]

    m = max(asf...,sol′...)
    @test asf / m ≈ sol′ / m atol=0.16

    # Plots
    #using Plots
    #plot(r_space, asf, seriestype = :scatter,label="Analytic solution")
    #plot!(r_space, sol′, label="Numeric solution")
    #savefig("MOL_NonLinear_Diffusion_1D_Test03.png")

end


function scale_devs(x, location = one(x), scale = 2one(x))
    x - (x - location) / scale
end

# Rescale the log catchability deviates, debias so that the exponentiated
# process has mean one, and exponentiate to get a multiplicative effect
"""
    transform_log_qdevs(log_qdevs, qdev_scale)

Given a vector of (potentially) normally distributed random variates, rescale
its standard deviation by a factor of `qdev_scale` and then exponentiate to
produce log-normally distributed random variates. Includes a bias correction so
that the mean of the resulting vector will have mean `exp(mean(log_qdevs))`.

    exp(qdev_scale * log_qdevs .- qdev_scale ^ 2 / 2)
"""
function transform_log_qdevs(log_qdevs, qdev_scale)
    exp.(qdev_scale * log_qdevs .- qdev_scale ^ 2 / 2)
end


g = CompleteDiGraph(5)
@test nv(g) == 5 && ne(g) == 20
g = CompleteGraph(5)
@test nv(g) == 5 && ne(g) == 10

g = CompleteBipartiteGraph(5, 8)
@test nv(g) == 13 && ne(g) == 40

g = StarDiGraph(5)
@test nv(g) == 5 && ne(g) == 4
g = StarGraph(5)
@test nv(g) == 5 && ne(g) == 4
g = StarGraph(1)
@test nv(g) == 1 && ne(g) == 0

g = PathDiGraph(5)
@test nv(g) == 5 && ne(g) == 4
g = PathGraph(5)
@test nv(g) == 5 && ne(g) == 4

g = CycleDiGraph(5)
@test nv(g) == 5 && ne(g) == 5
g = CycleGraph(5)
@test nv(g) == 5 && ne(g) == 5

g = WheelDiGraph(5)
@test nv(g) == 5 && ne(g) == 8
g = WheelGraph(5)
@test nv(g) == 5 && ne(g) == 8

g = Grid([3,3,4])
@test nv(g) == 3*3*4
@test ne(g) == 75
@test maximum(degree(g)) == 6
@test minimum(degree(g)) == 3

g = CliqueGraph(3,5)
@test nv(g) == 15 && ne(g) == 20
@test induced_subgraph(g,[1 2 3]) == CompleteGraph(3)

g = crosspath(3, BinaryTree(2))
# f = Vector{Vector{Int}}[[2 3 4];
# [1 5];
# [1 6];
# [1 5 6 7];
# [2 4 8];
# [3 4 9];
# [4 8 9];
# [5 7];
# [6 7]
# ]
I = [1,1,1,2,2,3,3,4,4,4,4,5,5,5,6,6,6,7,7,7,8,8,9,9]
J = [2,3,4,1,5,1,6,1,5,6,7,2,4,8,3,4,9,4,8,9,5,7,6,7]
V = ones(Int, length(I))
Adj = sparse(I,J,V)
@test Adj == sparse(g)

g = DoubleBinaryTree(3)
# [[3,2,8]
# [4,1,5]
# [1,6,7]
# [2]
# [2]
# [3]
# [3]
# [10,9,1]
# [11,8,12]
# [8,13,14]
# [9]
# [9]
# [10]
# [10]]
I = [1,1,1,2,2,2,3,3,3,4,5,6,7,8,8,8,9,9,9,10,10,10,11,12,13,14]
J = [3,2,8,4,1,5,1,6,7,2,2,3,3,10,9,1,11,8,12,8,13,14,9,9,10,10]
V = ones(Int, length(I))
Adj = sparse(I,J,V)
@test Adj == sparse(g)

rg3 = RoachGraph(3)
# [3]
# [4]
# [1,5]
# [2,6]
# [3,7]
# [4,8]
# [9,8,5]
# [10,7,6]
# [11,10,7]
# [8,9,12]
# [9,12]
# [10,11]
I = [ 1,2,3,3,4,4,5,5,6,6,7,7,7,8,8,8,9,9,9,10,10,10,11,11,12,12 ]
J = [ 3,4,1,5,2,6,3,7,4,8,9,8,5,10,7,6,11,10,7,8,9,12,9,12,10,11 ]
V = ones(Int, length(I))
Adj = sparse(I,J,V)
@test Adj == sparse(rg3)


include("../src/opti_module.jl")
include("../opti_test_functions/testfunctions.jl")

module otest
using BenchmarkTools
using ..opti
using ..optitestfun
using PyPlot
PyPlot.pygui(true)

include("test_opti_1d.jl")
include("test_line_search.jl")
include("test_gradient_descent.jl")
# include("accelerated_grad_descent.jl")

function test_bfgs()
    fcn(x) = optitestfun.rosenbrock(x, 4, 100)
    x0 = [-1.8, -2]

    (x, f, iter) = opti.quasi_newton(fcn, x0)
    println("Solution at: $(x)")
    println("Iter: $(iter)" )
end

function test_nesterov()

    fcn(x) = optitestfun.rosenbrock(x, 2)
    x0 = [-1.8, -2]

    (stat, opti_search_path) = opti.nesterov_momentum_gradient_descent(
        fcn,
        x0,
        tol_options = opti.ToleranceOptions(max_iter = 1000),
        log_path = true,
        method_parameters = opti.NesterovGradientDescentMethod(max_v = 0.1, beta = 0.4, alpha = 0.05)
        )

    show(stat)

    optitestfun.plot_rosenbrock([-2, 5], [-2, 5])
    PyPlot.plot( opti_search_path.x[1, :], opti_search_path.x[2, :], marker = :., markersize = 15 )
end

#=
# something is wrong with the implementation
function test_hyper_nesterov()

    fcn(x) = optitestfun.rosenbrock(x, 2)
    x0 = [-1.8, -2]

    (stat, opti_search_path) = opti.hyper_nesterov_momentum_gradient_descent(
        fcn,
        x0,
        tol_options = opti.ToleranceOptions(max_iter = 1000),
        log_path = true,
        method_parameters = opti.HyperNesterovGradientDescentMethod(max_v = 0.1, beta = 0.4, alpha = 0.05)
        )

    show(stat)

    optitestfun.plot_rosenbrock([-2, 5], [-2, 5])
    PyPlot.plot( opti_search_path.x[1, :], opti_search_path.x[2, :], marker = :., markersize = 15 )
end
=#

fcn(x) = optitestfun.rosenbrock(x, 2)
# x0 = [-1.8, -2]

# fcn(x) = sum(x .* x)
x0 = [-1.8, -2.0]
# t_sol = opti.bfgs(fcn, x0)
t_sol = opti.cg_pr(fcn, x0)

# t_qn = opti.quasi_newton(fcn, x0)

end # otest


"""
```julia
randDiagonal(d, n)

randDiagonal(n)
```
- `d` : default `Normal(0，1)`, entry distribution
- `n` : dimension

# Examples

Generates a 3 by 3 diagonal matrix, with non-zero elements from `Normal(0,1)`
```julia
randDiagonal(3)

3×3 Diagonal{Float64, Vector{Float64}}:
 0.440359   ⋅         ⋅
  ⋅        1.94832    ⋅
  ⋅         ⋅       -0.52536
```

Generates a 5 by 5 diagonal matrix, with non-zero elements from `Poisson(2)`
```julia
randDiagonal(Poisson(2),5)

5×5 Diagonal{Int64, Vector{Int64}}:
 1  ⋅  ⋅  ⋅  ⋅
 ⋅  0  ⋅  ⋅  ⋅
 ⋅  ⋅  0  ⋅  ⋅
 ⋅  ⋅  ⋅  3  ⋅
 ⋅  ⋅  ⋅  ⋅  3
```
"""
function randDiagonal(d::D, n::Int)  where D<:S
    Diagonal(rand(d,n))
end

randDiagonal(n::Int) =  randDiagonal(Normal(), n::Int)

"""
```julia
randTriangular(d , n ;  diag , Diag, upper )

randTriangular(n;diag, upper)
```
- `d` : entry distribution
- `n` : dimension
- `diag` : default `diag = d`, diagonal entry distribution
- `Diag` : default `Diag = true`, `true` includes diagonal, `false` with diagonal entries 0
- `upper` : default `upper = true`, `true` gives upper triangular, `false` gives lower triangular

# Examples
Generate an upper triangular matrix with entries Standard Normal
```julia
randTriangular(3)

3×3 UpperTriangular{Float64, Matrix{Float64}}:
 -0.572757  -0.459518   -1.60622
   ⋅         0.0216834  -0.416529
   ⋅          ⋅         -1.00807
```

Generate a 3 by 3 strictly lower triangular matrix, with nonzero entries uniform from ``\\{1,2,3\\}``
```julia
randTriangular(1:3,3,upper=false,Diag=false)

3×3 LowerTriangular{Int64, Transpose{Int64, Matrix{Int64}}}:
 0  ⋅  ⋅
 3  0  ⋅
 3  2  0
```
"""
function randTriangular(d::D, n::Int; diag=d::D, Diag=true, upper = true)  where D<:S
    M = UpperTriangular(zeros(eltype(d),n,n))
    for i in 1:n, j in i+1:n
        M[i,j] = rand(d)
    end
    if Diag
        M += randDiagonal(diag,n)
    end

    if !upper
        M = transpose(M)
    end
    M
end
randTriangular(n::Int; Diag=true, upper = true) = randTriangular(Normal(),n,Diag=Diag,upper=upper)

"""
```julia
randMatrix(d::D, n::Int, m = n::Int; norm = false::Bool) where D<:S

randMatrix(n::Int, m = n::Int; norm = false::Bool)
```
- `d` : entry distribution
- `n`,`m` : default `m = n` , dimensions
- `norm` : default `false`, if `norm` set to `true`, then the matrix will be normlaized with ``\\operatorname{min}(n,m)^{-1/2}``.
# Examples
Generates a 2 by 2 random  matrix with entries from the Standard  Gaussian.
```julia
randMatrix(2)

2×2 Matrix{Float64}:
 1.74043  -1.30317
 0.72765   0.639943
```

Generates a 3 by 2 random  matrix with entries uniformly from {1,2,3,...,10}.
```julia
randMatrix(1:10,3,2)

3×2 Matrix{Int64}:
  1  3
  6  4
 10  1
```
Generate a normalized random 2 by 2  Matrix with entries  `Poisson(2)` rvs.
Need to import the `Distributions` package for `Poisson(2)`
```julia
using Distributions
randMatrix(Poisson(2),2,norm = true)

2×2 Matrix{Float64}:
 1.41421   0.0
 0.707107  1.41421
```
"""
function randMatrix(d::D, n::Int, m = n::Int ; norm = false::Bool)  where D<:S

    M =  rand(d,n,m)

    if norm
         M/=sqrt(min(n,m))
    end

    return M
end

function randMatrix(n::Int, m = n::Int; norm = false::Bool)
    return randMatrix(Normal(), n,m ; norm = norm)
end


export bfgs

"""
bfgs(f,df,x)

BFGS method for solving min_x f(x)

"""
function bfgs(f::Function,df::Function,x::Vector;H=speye(length(x)), maxIter=20,atol=1e-8,out::Int=0,storeInterm::Bool=false,
	lineSearch::Function=(f,df,fk,dfk,xk,pk)->armijo(f,fk,dfk,xk,pk,maxIter=30))

    his = zeros(maxIter,3)
    I   = speye(length(x))
    X   = (storeInterm) ? zeros(length(x),maxIter) : []
    fk  = f(x)
    dfk = df(x)

    i = 1; flag = -1
    while i<=maxIter

        his[i,1:2] = [fk norm(dfk)]
        if storeInterm; X[:,i] = x; end;
        if norm(dfk)<atol
            his  = his[1:i,:]
            flag = 0
            break
        end

        # get search direction
        pk    = - H*dfk
        # line search
        ak,his[i,3] = lineSearch(f,df,fk,dfk,x,pk)
        if out>0
            @printf "iter=%4d\t|f|=%1.2e\t|df|=%1.2e\tLS=%d\n" i his[i,1] his[i,2] his[i,3]
        end
        if his[i,3]==-1
             flag = -3
             his  = his[1:i,:]
             break;
        end
        x    += ak*pk
        fk    = f(x)
        dfnew = df(x)
        sk    = ak*pk
        yk    = dfnew - dfk
        if dot(yk,sk)>0 # ensure that approximate Hessians remain positive definite
        	H     = (I - (sk*yk')/dot(sk,yk)) * H * (I - (yk*sk')/dot(sk,yk)) + (sk*sk')/dot(yk,sk)
        else
            if out>0
                warn("bfgs detected negative curvature. Resetting Hessian")
            end
            H = speye(length(x))
        end
        dfk  = dfnew
        i+=1
    end
    i = min(maxIter,i)

    if out>=0
        if flag==-1
            println(@sprintf("bfgs iterated maxIter (=%d) times but reached only atol of %1.2e instead of tol=%1.2e",i,his[i,2],atol))
        elseif flag==-3
            println(@sprintf("bfgs stopped of a line search fail at iteration %d.",i))
        elseif out>1
            println(@sprintf("bfgs achieved desired atol of %1.2e at iteration %d.",atol,i))
        end
    end

    if storeInterm; X = X[:,1:i]; end
    return x,flag,his,X,H
end


# # [Linear Time Continuous System](@id linear_continuous)
#
# Similar to the [`linear time discrete example`](@ref linear_discrete), we will now estimate a linear time continuous system ``\partial_t u = A u``.
# We simulate the correspoding system using `OrdinaryDiffEq.jl` and generate a [`ContinuousDataDrivenProblem`](@ref DataDrivenProblem) from the simulated data.

using DataDrivenDiffEq
using ModelingToolkit
using LinearAlgebra
using OrdinaryDiffEq
#md using Plots

A = [-0.9 0.2; 0.0 -0.2]
u0 = [10.0; -10.0]
tspan = (0.0, 10.0)

f(u,p,t) = A*u

sys = ODEProblem(f, u0, tspan)
sol = solve(sys, Tsit5(), saveat = 0.05);

# We could use the `DESolution` to define our problem, but here we want to use the data for didactic purposes.
# For a [`ContinuousDataDrivenProblem`](@ref DataDrivenProblem), we need either the state trajectory and the timepoints or the state trajectory and its derivate.

X = Array(sol)
t = sol.t
prob = ContinuousDataDrivenProblem(X, t)

# And plot the problems data.

#md plot(prob)

# We can see that the derivative has been automatically added via a [`collocation`](@ref collocation) method, which defaults to a `LinearInterpolation`.
# We can do a visual check and compare our derivatives with the interpolation of the `ODESolution`.

#md DX = Array(sol(t, Val{1}))
#md scatter(t, DX', label = ["Solution" nothing], color = :red, legend = :bottomright)
#md plot!(t, prob.DX', label = ["Linear Interpolation" nothing], color = :black)

# Since we have a linear system, we can use `gDMD`, which approximates the generator of the dynamics

#md res = solve(prob, DMDSVD())
#md println(res)

# We see that the system has been recovered correctly, indicated by the small error and high AIC score of the result. We can confirm this by looking at the resulting [`Basis`](@ref)

#md system = result(res)
#md println(system)

# And also plot the prediction of the recovered dynamics

#md plot(res)

# Or a have a look at the metrics of the result

#md metrics(res)

# And check the parameters of the result

#md parameters(res)

# or the generator of the system

#md Matrix(generator(system))

# to see that the operator is slightly off, but within expectations.
# In a real example, this could have many reasons, e.g. noisy data, insufficient time samples or missing states.

# Sticking to the same procedure as earlier, we now use a linear sparse regression to solve the problem

using ModelingToolkit

@parameters t
@variables x[1:2](t)

basis = Basis(x, x, independent_variable = t, name = :LinearBasis)
#md print(basis) #hide

# Afterwards, we simply `solve` the already defined problem with our `Basis` and a `SparseOptimizer`

sparse_res = solve(prob, basis, STLSQ(1e-1))
#md println(sparse_res)

# Which holds the same equations
sparse_system = result(sparse_res)
#md println(sparse_system)

# Again, we can have a look at the result

#md plot(
#md     plot(prob), plot(sparse_res), layout = (1,2)
#md )

# Both results can be converted into an `ODESystem`

@named sys = ODESystem(
    equations(sparse_system),
    get_iv(sparse_system),
    states(sparse_system),
    parameters(sparse_system)
    );

# And simulated using `OrdinaryDiffEq.jl` using the (known) initial conditions and the parameter mapping of the estimation.

x0 = [x[1] => u0[1], x[2] => u0[2]]
ps = parameter_map(sparse_res)

ode_prob = ODEProblem(sys, x0, tspan, ps)
estimate = solve(ode_prob, Tsit5(), saveat = prob.t);

# And look at the result
#md plot(sol, color = :black)
#md plot!(estimate, color = :red, linestyle = :dash)

#md # ## [Copy-Pasteable Code](@id linear_continuous_copy_paste)
#md #
#md # ```julia
#md # @__CODE__
#md # ```

@test all(aic(sparse_res) .>= 1e3) #src
@test all(l2error(sparse_res) .<= 5e-1) #src
@test all(determination(sparse_res) .>= 0.97) #src
@test Array(sol) ≈ Array(estimate) rtol = 5e-2 #src


using Dates

# Test data is 3 days after a starting reference on Jan 1:
# The actual phi phase is [0, 1, 2, 3]

geolist = [Date(2019, 1, 1), Date(2019, 1, 3), Date(2019, 1, 6), Date(2019, 1, 10)]
timediffs = InsarTimeseries.day_diffs(geolist)
@test timediffs == [2, 3, 4]

intlist = [(geolist[i], geolist[j]) for i = 1:length(geolist)-1 for j = i+1:length(geolist)]

# julia> intlist
#  (2019-01-01, 2019-01-03)
#  (2019-01-01, 2019-01-06)
#  (2019-01-01, 2019-01-10)
#  (2019-01-03, 2019-01-06)
#  (2019-01-03, 2019-01-10)
#  (2019-01-06, 2019-01-10)

A = InsarTimeseries.build_A_matrix(geolist, intlist)
@test A == [
    1 0 0
    0 1 0
    0 0 1
    -1 1 0
    -1 0 1
    0 -1 1
]

B = InsarTimeseries.build_B_matrix(geolist, intlist)
@test B ≈ [
    2.0 0.0 0.0
    2.0 3.0 0.0
    2.0 3.0 4.0
    0.0 3.0 0.0
    0.0 3.0 4.0
    0.0 0.0 4.0
]

truth_phi = [0.0, 1.0, 2.0, 3.0]
truth_v = [1 / 2, 1 / 3, 1 / 4]
dphis = [1.0, 2.0, 3.0, 1.0, 2.0, 1.0]
# Since we add no noise here, we should perfectly recover the phi phase
@test isapprox(A \ dphis, truth_phi[2:end], rtol = 1e-7)

v_hat = B \ dphis
@test isapprox(v_hat, truth_v, rtol = 1e-7)

phi_hat = InsarTimeseries.integrate_velocities(v_hat, timediffs)
@test isapprox(phi_hat, truth_phi, rtol = 1e-7)

dpr = dphis .+ rand(eltype(dphis), size(dphis))
@test isapprox(
    A \ dpr,
    InsarTimeseries.integrate_velocities(B \ dpr, timediffs)[2:end],
    rtol = 1e-7,
)
# TODO: make the const. vel into a function to test it


"""
    atmosphere_isa(height)
Calculate temperature, pressure, density and sound velocity for the given geopotential
height according to International Standard Atmosphere 1976.

# References
1.U.S. Standard Atmosphere, 1976, U.S. Government Printing Office, Washington, D.C., 1976

| Layer | h (m) | p (Pa)  | T (K)  | ``α`` (K/m) |
| ----- | ----- | ------- | ------ | ----------- |
| 0     | 0     | 101325  | 288.15 | -0.0065     |
| 1     | 11000 | 22632.1 | 216.65 | 0           |
| 2     | 20000 | 5474.89 | 216.65 | 0.001       |
| 3     | 32000 | 868.019 | 228.65 | 0.0028      |
| 4     | 47000 | 110.906 | 270.65 | 0           |
| 5     | 51000 | 66.9389 | 270.65 | -0.0028     |
| 6     | 71000 | 3.95642 | 214.65 | -0.002      |

Source: [https://en.wikipedia.org/wiki/U.S._Standard_Atmosphere](https://en.wikipedia.org/wiki/U.S._Standard_Atmosphere)

"""
function atmosphere_isa(height)

    if 0 <= height < 11000  # Troposphere
        α = -0.0065  # K/m
        T0 = 288.15      # K
        p0 = 101325.0    # Pa

        T = T0 + α * height
        p = p0 * (T0 / T) ^ (gD / (R_AIR * α))

    elseif 11000 <= height < 20000  # Tropopause
        T = 216.65    # K
        p0 = 22632.1  # Pa
        h0 = 11000    # m

        p = p0 * exp(-gD * (height - h0) / (R_AIR * T))

    elseif 20000 <= height < 32000  # Stratosphere 1
        α = 0.001    # K/m
        T0 = 216.65      # K
        p0 = 5474.89     # Pa
        h0 = 20000       # m

        T = T0 + α * (height - h0)
        p = p0 * (T0 / T) ^ (gD / (R_AIR * α))

    elseif 32000 <= height < 47000  # Stratosphere 2
        α = 0.0028   # K/m
        T0 = 228.65      # K
        p0 = 868.019     # Pa
        h0 = 32000       # m

        T = T0 + α * (height - h0)
        p = p0 * (T0 / T) ^ (gD / (R_AIR * α))

    elseif 47000 <= height < 51000  # Stratopause
        T = 270.65    # K
        p0 = 110.906  # Pa
        h0 = 47000    # m
        h0 = 47000    # m

        p = p0 * exp(-gD * (height - h0) / (R_AIR * T))

    elseif 51000 <= height < 71000  # Mesosphere 1
        α = -0.0028  # K/m
        T0 = 270.65      # K
        p0 = 66.9389     # Pa
        h0 = 51000       # m

        T = T0 + α * (height - h0)
        p = p0 * (T0 / T) ^ (gD / (R_AIR * α))

    elseif 71000 <= height <= 84500  # Mesosphere 2
        α = -0.002   # K/m
        T0 = 214.65      # K
        p0 = 3.95642     # Pa
        h0 = 71000       # m

        T = T0 + α * (height - h0)
        p = p0 * (T0 / T) ^ (gD / (R_AIR * α))

    else
        throw(DomainError(height, "height must be between 0 m and 84500 m"))
    end

    rho = p / (R_AIR * T)
    a   = sqrt(γ_AIR * R_AIR * T)

    return [T, p, rho, a]
end


"""
    modular_inverse(base::Int64, modulus::Int64)

Compute the multiplicative inverse of the `base` under the `modulus`.

# Arguments
- `base::Int64`: base
- `modulus ::Int64`: modulus

# Examples
```julia-repl
julia> import ClassicAlgorithmsCollections
julia> ClassicAlgorithmsCollections.modular_inverse(3, 11)
4
```
"""
function modular_inverse(base::Int64, modulus::Int64)
    base = base % modulus
    for i in 1:modulus
        if (base * i) % modulus == 1
            return i
        end
    end
    return 1
end


"""
    taylor(df,a,b,y0,M)

berisi program untuk mencari solusi persamaan differensial `y'=f(t,y)`
dengan masalah nilai awal `y(a) = y0` pada interval `[a, b]`. Program ini secara default
berisi 5 masukan, yaitu fungsi `df` yang berisi nilai dari fungsi y', y'', y''', y'''',
titik ujung interval penyelesaian `[a,b]`, nilai awal
`y0`, dan jumlah sub-interval `M`.
# Examples
```jldoctest
julia> function df(t,y)
         y1 = (t-y)/2;
         y2 = (2-t+y)/4;
         y3 = (-2+t-y)/8;
         y4 = (2-t+y)/16;
         z = [y1,y2,y3,y4];
       end

julia> sol1 = taylor(df,0,3,1,6)
7×2 Array{Float64,2}:
 0.0  1.0
 0.5  0.836426
 1.0  0.819628
 1.5  0.917142
 2.0  1.10368
 2.5  1.35956
 3.0  1.66943
```
return solusi masalah nilai awal `sol`.
"""
function taylor(df,a,b,y0,M)
  M = Int(M)
  h = (b-a)/M;
  T = a:h:b;
  Y = Array{Float64}(undef,length(T),1)
  Y[1]=y0;
  for j = 1:M
    D = df(T[j],Y[j]);
    Y[j+1]=Y[j]+h*(D[1]+h*(D[2]/2+h*(D[3]/6+h*D[4]/24)));
  end
  sol = [T Y];
end


"expected_arc(i,n) this polynomial indicates the expected number of NN-arcs that would pass over index `i` for a series of length`n` if the series was completely random. This is used to normalize the `segment_profile` to account for bias towards the edges."
expected_arc(i,n) = 2*(-i^2*n/(n-1)^2 + i*n/(n-1)) # polynomial coefficients solved for by using constraints e(0) = 0, e(n-1) = 0, e(n/2) = n/2

"""
    segment_profile(p::Profile)

Calculate the MP-index of a matrix profile. This index has the same length as the profile, and tells you how many nearest-neighbor arcs passes over index `i`. It's normalized by the expected number of arcs for a completely random time series, so that a value of 1 indicates a poor segmentation point, and a value close to 0 indicates a likely segmentation point.
"""
function segment_profile(p::Profile)
    I = p.I
    n = length(I)
    mpi = zeros(eltype(p.P), n)
    for i in 1:n
        Ii = I[i]
        if Ii > i
            for j = i:Ii-1
                mpi[j] += 1
            end
        else
            for j = Ii+1:i
                mpi[j] += 1
            end
        end
    end
    mpi .= min.(mpi ./ expected_arc.(0:n-1, n), 1)
    mpi
end

"""
    i = segment(p::Profile)

Returns an index `i` indicating the most likely segmentation point of profile `p`, i.e., the point which the fewst nearest-neighbor arcs passes over.

Ref: Matrix Profile VIII: Domain Agnostic Online Semantic
Segmentation at Superhuman Performance Levels
"""
function segment(p::Profile)
    argmin(segment_profile(p))
end


n = 0
while n < 10
    global n; n += 1
    println(n)
end

for n in 1:10
    println(n)
end

for friend in ["Ted", "Robyn", "Barney","Lily","Marshall"]
    println("Hi $friend, it's great to see you!")
end

#=
let's use for loops to create some addition tables,
where the value of every entry is the sum of its row and column indices.
=#

function create_table1(m,n)
    A = fill(0, (m, n))
    for i in 1:m
        for j in 1:n
            A[i,j] = i+j
        end
    end
    return A
end

#=
Here is some syntactic sugar for the same nested `for` loop
=#

function create_table2(m,n)
    A = fill(0, (m, n))
    for i in 1:m, j in 1:n
        A[i,j] = i+j
    end
    return A
end

#=
The more "Julia" way to create this addition table would have been
with an array comprehension.
=#

function create_table3(m,n)
    C = [i+j for i in 1:m, j in 1:n]
    return C
end

@show create_table1(4,5)
@show create_table2(4,5)
@show create_table3(4,5)

using BenchmarkTools
m,n=1000,1000
@btime create_table1(m,n)
@btime create_table2(m,n)
@btime create_table3(m,n)


"""
naive implementation
"""
function naiveform!(C, A, B)
    row, col  = size(C)
    colA = size(A)[2]
    for i in 1:row
        for j in 1:col
            for k in 1:colA
                C[i, j] = C[i, j] + A[i, k] * B[k, j]
            end
        end
    end
    C
end

"""
inner-product-form
"""
function ipform!(C, A, B)
    row, col  = size(C)
    colA = size(A)[2]
    for i in 1:row
        for j in 1:col
            dataC = 0.0
            for k in 1:colA
                dataC = dataC + A[i, k] * B[k, j]
            end
            C[i, j] = dataC
        end
    end
end

"""
outer-product-form
"""
function opform!(C, A, B)
    row, col  = size(C)
    colA = size(A)[2]
    for k in 1:colA
        for j in 1:col
            dataB = B[k, j]
            for i in 1:row
                C[i, j] = C[i, j] + A[i, k] * dataB
            end
        end
    end
end

"""
middle-product-form
"""
function mpform!(C, A, B)
    row, col  = size(C)
    colA = size(A)[2]
    for j in 1:col
        for k in 1:colA
            dataB = B[k, j]
            for i in 1:row
                C[i, j] = C[i, j] + A[i, k] * dataB
            end
        end
    end
end


# Classical Multidimensional Scaling

## convert Gram matrix to Distance matrix

function gram2dmat!{DT}(D::AbstractMatrix{DT}, G::AbstractMatrix)
    # argument checking
    m = size(G, 1)
    n = size(G, 2)
    m == n || error("D should be a square matrix.")
    size(D) == (m, n) ||
        throw(DimensionMismatch("Sizes of D and G do not match."))

    # implementation
    for j = 1:n
        for i = 1:j-1
            @inbounds D[i,j] = D[j,i]
        end
        D[j,j] = zero(DT)
        for i = j+1:n
            @inbounds D[i,j] = sqrt(G[i,i] + G[j,j] - 2 * G[i,j])
        end
    end
    return D
end

gram2dmat{T<:Real}(G::AbstractMatrix{T}) = gram2dmat!(similar(G, Base.momenttype(T)), G)

## convert Distance matrix to Gram matrix

function dmat2gram!{GT}(G::AbstractMatrix{GT}, D::AbstractMatrix)
    # argument checking
    m = size(D, 1)
    n = size(D, 2)
    m == n || error("D should be a square matrix.")
    size(G) == (m, n) ||
        throw(DimensionMismatch("Sizes of G and D do not match."))

    # implementation
    u = zeros(GT, n)
    s = 0.0
    for j = 1:n
        s += (u[j] = Base.sumabs2(view(D,:,j)) / n)
    end
    s /= n

    for j = 1:n
        for i = 1:j-1
            @inbounds G[i,j] = G[j,i]
        end
        for i = j:n
            @inbounds G[i,j] = (u[i] + u[j] - abs2(D[i,j]) - s) / 2
        end
    end
    return G
end

dmat2gram{T<:Real}(D::AbstractMatrix{T}) = dmat2gram!(similar(D, Base.momenttype(T)), D)

## classical MDS

function classical_mds{T<:Real}(D::AbstractMatrix{T}, p::Integer)
    n = size(D, 1)
    p < n || error("p must be less than n.")

    G = dmat2gram(D)
    E = eigfact!(Symmetric(G))
    ord = sortperm(E.values; rev=true)
    (v, U) = extract_kv(E, ord, p)
    for i = 1:p
        @inbounds v[i] = sqrt(v[i])
    end
    scale!(U, v)'
end


#
# Copyright (c) 2021 Tobias Thummerer, Lars Mikelsons
# Licensed under the MIT license. See LICENSE file in the project root for details.
#

using FMIExport

FMU_FCT_INIT = function()
    m = 1.0         # ball mass
    r = 0.0         # ball radius
    d = 0.7         # ball collision damping
    v_min = 1e-1    # ball minimum velocity
    g = -9.81       # gravity constant

    s = 1.0         # ball position
    v = 0.0         # ball velocity
    a = 0.0         # ball acceleration

    t = 0.0
    x = [s, v]
    ẋ = [v, a]
    u = []
    p = [m, r, d, v_min, g]

    return (t, x, ẋ, u, p)
end

FMU_FCT_EVALUATE = function(t, x, ẋ, u, p)
    m, r, d, v_min, g = (p...,)
    s, v = (x...,)
    _, a = (ẋ...,)

    if s <= r && v < 0.0
        s = r
        v = -v*d

        # stop bouncing to prevent high frequency bouncing (and maybe tunneling the floor)
        if v < v_min
            v = 0.0
            g = 0.0
        end
    end

    a = (m * g) / m     # the system's physical equation

    x = [s, v]
    ẋ = [v, a]
    p = [m, r, d, v_min, g]

    return (x, ẋ, p)
end

FMU_FCT_OUTPUT = function(t, x, ẋ, u, p)
    m, r, d, v_min, g = (p...,)
    s, v = (x...,)
    _, a = (ẋ...,)

    y = [s]

    return y
end

FMU_FCT_EVENT = function(t, x, ẋ, u, p)
    m, r, d, v_min, g = (p...,)
    s, v = (x...,)
    _, a = (ẋ...,)

    z1 = (s-r)              # event 1: ball hits ground

    if s==r && v==0.0
        z1 = 1.0            # event 1: ball stay-on-ground
    end

    z = [z1]

    return z
end

# this function is called, as soon as the DLL is loaded and Julia is initialized
# must return a FMU2-instance to work with
FMIBUILD_CONSTRUCTOR = function(resPath="")
    fmu = fmi2CreateSimple(initializationFct=FMU_FCT_INIT,
                        evaluationFct=FMU_FCT_EVALUATE,
                        outputFct=FMU_FCT_OUTPUT,
                        eventFct=FMU_FCT_EVENT)

    # states [2]
    fmi2AddStateAndDerivative(fmu, "ball.s"; stateDescr="Absolute position of ball center of mass", derivativeDescr="Absolute velocity of ball center of mass")
    fmi2AddStateAndDerivative(fmu, "ball.v"; stateDescr="Absolute velocity of ball center of mass", derivativeDescr="Absolute acceleration of ball center of mass")

    # outputs [1]
    fmi2AddRealOutput(fmu, "ball.s"; description="Absolute position of ball center of mass")

    # parameters [5]
    fmi2AddRealParameter(fmu, "m";     description="Mass of ball")
    fmi2AddRealParameter(fmu, "r";     description="Radius of ball")
    fmi2AddRealParameter(fmu, "d";     description="Collision damping constant (velocity fraction after hitting the ground)")
    fmi2AddRealParameter(fmu, "v_min"; description="Minimal ball velocity to enter on-ground-state")
    fmi2AddRealParameter(fmu, "g";     description="Gravity constant")

    fmi2AddEventIndicator(fmu)

    return fmu
end

### FMIBUILD_NO_EXPORT_BEGIN ###
# The line above is a start-marker for excluded code for the FMU compilation process!

tmpDir = mktempdir(; prefix="fmibuildjl_test_", cleanup=false)
@info "Saving example files at: $(tmpDir)"
fmu_save_path = joinpath(tmpDir, "BouncingBall.fmu")

fmu = FMIBUILD_CONSTRUCTOR()
using FMIBuild: fmi2Save        # <= this must be excluded during export, because FMIBuild cannot execute itself (but it is able to build)
fmi2Save(fmu, fmu_save_path)    # <= this must be excluded during export, because fmi2Save would start an infinte build loop with itself

### some tests ###
# using FMI
# comp = fmiInstantiate!(fmu; loggingOn=true)
# solution = fmiSimulateME(comp, 0.0, 10.0; dtmax=0.1)
# fmiPlot(fmu, solution)
# fmiFreeInstance!(comp)

# The following line is a end-marker for excluded code for the FMU compilation process!
### FMIBUILD_NO_EXPORT_END ###


const singular = let res_init=zeros(4), jac_init=zeros(4,4), x_init=[3.0, -1.0, 0.0, 1.0]

    function res(x, r)
        r[1] = x[1] + 10.0 * x[2]
        r[2] = sqrt(5.0) * (x[3] - x[4])
        r[3] = (x[2] - 2x[3])^2
        r[4] = sqrt(10.0) * (x[1] - x[4])^2
        return r
    end

    function jac(x, jac)
        jac[:] .= 0.0
        jac[1, 1] = 1.0
        jac[1, 2] = 10.0
        jac[2, 3] = sqrt(5.0)
        jac[2, 4] = -jac[2, 3]
        jac[3, 2] = 2.0 * (x[2] - 2.0 * x[3])
        jac[3, 3] = -2.0 * jac[3, 2]
        jac[4, 1] = sqrt(40.0) * (x[1] - x[4])
        jac[4, 4] = -jac[4, 1]
        return jac
    end

    f(;scale=1, verbose=false, print_steps=false) = testone("singular", res, jac,
                                                   res_init, jac_init, x_init;
                                                   scale=scale, verbose=verbose,
                                                   print_steps=print_steps)
end


using Distributions

"""
General Interface for Computation of confidence interval of price

		IC=confinter(mcProcess,rfCurve,mcBaseData,payoff_,alpha)

Where:\n
		mcProcess          = Process to be simulated.
		rfCurve  = Zero Rate Data.
		mcBaseData = Basic properties of MonteCarlo simulation
		payoff_ = Payoff(s) to be priced
		alpha [Optional, default to 99%] = confidence level

		Price     = Price of the derivative

"""
function confinter(mcProcess::BaseProcess, rfCurve::AbstractZeroRateCurve, mcConfig::MonteCarloConfiguration, abstractPayoff::AbstractPayoff, alpha::Real = 0.99)
    set_seed(mcConfig)
    T = maturity(abstractPayoff)
    Nsim = mcConfig.Nsim
    S = simulate(mcProcess, rfCurve, mcConfig, T)
    Payoff = payoff(S, abstractPayoff, rfCurve)
    mean1 = mean(Payoff)
    var1 = var(Payoff)
    alpha_ = 1 - alpha
    dist1 = Distributions.TDist(Nsim)
    tstar = quantile(dist1, 1 - alpha_ / 2.0)
    IC = (mean1 - tstar * sqrt(var1) / Nsim, mean1 + tstar * sqrt(var1) / Nsim)
    return IC
end

function confinter(mcProcess::BaseProcess, rfCurve::AbstractZeroRateCurve, mcConfig::MonteCarloConfiguration, abstractPayoffs::Array{abstractPayoff_}, alpha::Real = 0.99) where {abstractPayoff_ <: AbstractPayoff}
    set_seed(mcConfig)
    maxT = maximum([maturity(abstractPayoff) for abstractPayoff in abstractPayoffs])
    Nsim = mcConfig.Nsim
    S = simulate(mcProcess, rfCurve, mcConfig, maxT)
    Means = [mean(payoff(S, abstractPayoff, rfCurve, maxT)) for abstractPayoff in abstractPayoffs]
    Vars = [var(payoff(S, abstractPayoff, rfCurve, maxT)) for abstractPayoff in abstractPayoffs]
    alpha_ = 1 - alpha
    dist1 = Distributions.TDist(Nsim)
    tstar = quantile(dist1, 1 - alpha_ / 2.0)
    IC = [(mean1 - tstar * sqrt(var1) / Nsim, mean1 + tstar * sqrt(var1) / Nsim) for (mean1, var1) in zip(Means, Vars)]
    return IC
end


using vOptGeneric
using GLPK,GLPKMathProgInterface
m = vModel(solver = GLPKSolverMIP())
# using CPLEX
# m = vModel(solver = CplexSolver())

@variable(m, x[1:6,1:6], Bin)

M = 50
p1 =   [M 4 5 M M M;
        M M 2 1 2 7;
        M M M 5 2 M;
        M M 5 M M 3;
        M M M M M 4;
        M M M M M M]
p2 =   [M 3 1 M M M;
        M M 1 4 2 2;
        M M M 1 7 M;
        M M 1 M M 2;
        M M M M M 2;
        M M M M M M]

@addobjective(m, Min , sum(x[i,j]*p1[i,j] for i=1:6, j=1:6))
@addobjective(m, Min , sum(x[i,j]*p2[i,j] for i=1:6, j=1:6))

@constraint(m, eps, sum(x[1,j] for j = 1:6) == 1)
@constraint(m, sum(x[i,6] for i = 1:6) == 1)
@constraint(m, cstr[i=2:5], sum(x[i,j] for j = 1:6) - sum(x[j,i] for j =1:6) == 0)

solve(m, method=:epsilon, step=0.5)

#get the results and print/plot them
Y_N = getY_N(m)

for n = 1:length(Y_N)
    X = getvalue(x, n)
    for ind in find(X)
        i,j = ind2sub(X,ind)
        print("$i->$j ")
    end
    println("| z = ",Y_N[n])
end

# using PyPlot
# f1 = map(y -> y[1], Y_N)
# f2 = map(y -> y[2], Y_N)
# xlabel("z1")
# ylabel("z2")
# plot(f1,f2,"bx", markersize = "6")
# !isinteractive() && show()


"""
    centroids_radii(points::Array{Float64, 2}, indices_simplices::Array{Int, 1})
Compute the centroids and radii of the simplices in the triangulation in any dimension.

`points::Array{Float64, 2}` Points furnishing the triangulation (size = npoints x dim).
`indices_simplices::Array{Int, 1}` Indices of the vertices furnishing simplices of the
    triangulation. Each row refers to one simplex.
"""
function centroids_radii2(points, indices_simplices)
    dim = size(points, 2)
    nsimplices = size(indices_simplices, 1)
    centroids = zeros(nsimplices, dim)
    radii = zeros(nsimplices)

    for i = 1:nsimplices
        simplex = points[indices_simplices[i, :], :] # (dim + 1) x dim)
        centroid = sum(simplex, dims=1) / (dim + 1)
        centroid_matrix = repeat(centroid, dim + 1, 1)

        # Subtract centroid from each simplex
        subtracted = simplex - centroid_matrix
        radius = sqrt(maximum(sum(subtracted.^2, dims=2)))

        centroids[i, :] = centroid
        radii[i] = radius

    end

    return centroids, radii
end