#Function that group the mainly statistics moments that include nan values

function nanmean(vector::Vector)
N=length(vector);
s=0.0;                                        #--sample mean sum
m=0.0;                                        #--counter for the sample size
  for i=1:N
     if isnan(vector[i])==0;                  #--make sure to sum numeric values
        m=m+1;
        s=s+vector[i];
     end
  end
mu=s/m;
mu
end

#In nanvar.jl
#https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Higher-order_statistics
function nanvar(vector::Array)
N=length(vector);
m=0.0;             #---Initialize counter for samples
mu=0.0;            #---Initialize mean
M2=0.0;            #---Intitialize sum(x_i-mu)^2

 for i=1:N
      if isnan(vector[i])==0;                  #--make sure to sum numeric values
         m=m+1;
         delta=vector[i]-mu;
         mu=mu+delta/m;
         M2=M2+delta*(vector[i]-mu);
      end
 end
 if m<2;
    return NaN
 else
    return M2/m
 end
end

#In nanskew.jl
#https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Higher-order_statistics
function nanskew(vector::Array)
N=length(vector);
m=0.0;             #---Initialize counter for samples
mu=0.0;            #---Initialize mean
M2=0.0;            #---Intitialize sum(x_i-mu)^2
M3=0.0;            #---Intitialize sum(x_i-mu)^2
M4=0.0;            #---Intitialize sum(x_i-mu)^2

 for i=1:N
      if isnan(vector[i])==0;                  #--make sure to sum numeric values
         n1=m;
         m=m+1;
         delta=vector[i]-mu;
         delta_m=delta/m;
         delta_m2=delta_m*delta_m;
         term1=delta*delta_m*n1;
         mu=mu+delta_m;
         M3=M3+term1*delta_m*(m-2)-3*delta_m*M2
         M2=M2+term1;
      end
 end
skewness=sqrt(m)*M3/M2^(3/2);
return skewness
end

#In nankurt.jl
#https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Higher-order_statistics
function nankurt(vector::Array)
N=length(vector);
m=0.0;             #---Initialize counter for samples
mu=0.0;            #---Initialize mean
M2=0.0;            #---Intitialize sum(x_i-mu)^2
M3=0.0;            #---Intitialize sum(x_i-mu)^2
M4=0.0;            #---Intitialize sum(x_i-mu)^2

 for i=1:N
      if isnan(vector[i])==0;                  #--make sure to sum numeric values
         n1=m;
         m=m+1;
         delta=vector[i]-mu;
         delta_m=delta/m;
         delta_m2=delta_m*delta_m;
         term1=delta*delta_m*n1;
         mu=mu+delta_m;
         M4=M4+term1*delta_m2*(m*m-3*m+3)+6*delta_m2*M2-4*delta_m*M3
         M3=M3+term1*delta_m*(m-2)-3*delta_m*M2
         M2=M2+term1;
      end
 end
#kurtosis=(m*M4)/(M2*M2)-3;      #--normalized
kurtosis=(m*M4)/(M2*M2);
return kurtosis
end


using Test

import LinearAlgebra.diagm
import LinearAlgebra.triu
import LinearAlgebra.I
import LinearAlgebra.det

include("../../../lib/KTBC.jl")
import .KTBC.CreateKTBC

function createU(dim=3)
    return I(dim) - diagm(1 => ones(dim - 1))
end

function createS(dim=3)
    return triu(ones(dim, dim))
end

# 3
U₅ = createU(5)
S₅ = createS(5)

U₅ * S₅

@test S₅ == inv(U₅)

# 4
S₄ = triu(ones(4, 4))

S₄ * S₄'

# 5
KTBC₂ = CreateKTBC()
K₂ = convert(Array{Int}, KTBC₂[1])

det(K₂)
inv(K₂)

@test inv(K₂) * det(K₂) == [
  2.0 1.0
  1.0 2.0
]

KTBC₃ = CreateKTBC(3)
K₃ = convert(Array{Int}, KTBC₃[1])

det(K₃)
inv(K₃)

@test round.(Int, inv(K₃) * det(K₃)) == [
  3 2 1
  2 4 2
  1 2 3
]

# 6
function klcounter(n=4)
    j = 1
    result = zeros(n, n)
    while j <= n
        i = j
        while i <= n
            result[i, j] = (n - i + 1) * j
            i += 1
        end
        j += 1
    end
    return result
end

@test klcounter() == [
  4.0  0.0  0.0  0.0
  3.0  6.0  0.0  0.0
  2.0  4.0  6.0  0.0
  1.0  2.0  3.0  4.0
]

# 7
KTBC₃ = CreateKTBC(3)

T₃ = convert(Array{Int}, KTBC₃[2])
K₃ - T₃

u = [
  1
  0
  0
]
vᵀ = transpose(u)

@test u * vᵀ == K₃ - T₃

invT₃ = inv(T₃)
invK₃ = inv(K₃)

@test round.(Int, 4(invT₃ - invK₃)) == [
 9  6  3
 6  4  2
 3  2  1
]

u = [
  3
  2
  1
]
vᵀ = transpose(u)

@test 1 / 4 * u * vᵀ == round.(invT₃ - invK₃; digits=2)


# This file investigates MSE with respect to free variables

using FractalTools
using GeometryBasics
using Makie

# Construct interpolation data
f = FractalTools.ackley
α = 10
ngon = [BigFloat.([-1., -1.]), BigFloat.([1., -1]), BigFloat.([0., 1.])] * α
npts = 100
pts = getdata(f, ngon, npts)

# Construct test data
tpts = getdata(ngon, npts)
ntpts = length(tpts)

# Compute errors
fvals = map(pt -> f(pt...), tpts)
freevars = 0.001 : 0.001 : 0.05
mse = map(freevars) do freevar
    interp = interpolate(pts, Interp2D(freevar))
    ivals = map(pt -> interp(pt...), tpts)
    sum((fvals - ivals).^2) / ntpts
end

# Plot mse
fig = Figure()
ax = fig[1, 1] = Axis(fig, xlabel="Free Variable", ylabel="MSE", title="2D Interpolation MSE")
stem!(ax, freevars, mse, color=:black)
save(joinpath(@__DIR__, "interp2d_mse.png"), fig)
display(fig)

# using FractalTools
# using GeometryBasics
# using Makie

# # Construct interpolation data
# f(x, y) = -20*exp(-0.2 * √(0.5*(x^2+y^2))) -
#             exp(0.5*(cos(2π*x)+cos(2π*y))) +
#             MathConstants.e + 20

# # f(x, y) = x^2 + y^2 + 1

# # f(x, y) = sin(x) * cos(y)
# ngon = Triangle(
#     Point(BigFloat(-5.), BigFloat(-5.)),
#     Point(BigFloat(5), BigFloat(-5.)),
#     Point(BigFloat(0), BigFloat(5.)))

# # ngon = [BigFloat.([-5,5]),
# #         BigFloat.([-5,-5]),
# #         BigFloat.([5,-5]),
# #         BigFloat.([5,5])
# # ]
# npts = 100
# pts = getdata(f, ngon, npts)

# # Construct test data
# tpts = getdata(ngon, npts)
# ntpts = length(tpts)

# # Compute errors
# fvals = map(pt -> f(pt...), tpts)
# freevars = 0.001 : 0.001 : 0.05
# mse = map(freevars) do freevar
#     interp = interpolate(pts, Interp2D(freevar))
#     ivals = map(pt -> interp(pt...), tpts)
#     sum((fvals - ivals).^2) / ntpts
# end

# # Plot mse
# fig = Figure()
# ax = fig[1, 1] = Axis(fig, xlabel="Free Variable", ylabel="MSE", title="2D Interpolation MSE")
# stem!(ax, freevars, mse, color=:black)
# save(joinpath(@__DIR__, "interp2d_mse.png"), fig)
# display(fig)


include("misc.jl")

function findMin(funObj,w;maxIter=100,epsilon=1e-2,derivativeCheck=false,verbose=true)
	# funObj: function that returns (objective,gradient)
	# w: initial guess
	# maxIter: maximum number of iterations
	# epsilon: stop if the gradient gets below this
	# derivativeCheck: whether to check against numerical gradient

	# Evalluate the intial objective and gradient
	(f,g) = funObj(w)

	# Optionally check if gradient matches finite-differencing
	if derivativeCheck
		g2 = numGrad(funObj,w)

		if maximum(abs.(g-g2)) > 1e-4
			@show([g g2])
			@printf("User and numerical derivatives differ\n")
			sleep(1)
		else
			@printf("User and numerical derivatives agree\n")
		end
	end

	# Initial step size and sufficient decrease parameter
	gamma = 1e-4
	alpha = 1
	for i in 1:maxIter

		# Try out the current step-size
		wNew = w - alpha*g
		(fNew,gNew) = funObj(wNew)

		# Decrease the step-size if we increased the function
		gg = dot(g,g)
		while fNew > f - gamma*alpha*gg

			if verbose
				@printf("Backtracking\n")
			end

			# Fit a degree-2 polynomial to set step-size
			alpha = alpha^2*gg/(2(fNew - f + alpha*gg))

			# Try out the smaller step-size
			wNew = w - alpha*g
			(fNew,gNew) = funObj(wNew)
		end

		# Guess the step-size for the next iteration
		y = gNew - g
		alpha *= -dot(y,g)/dot(y,y)

		# Sanity check on the step-size
		if (!isfinitereal(alpha)) | (alpha < 1e-10) | (alpha > 1e10)
			alpha = 1
		end

		# Accept the new parameters/function/gradient
		w = wNew
		f = fNew
		g = gNew

		# Print out some diagnostics
		gradNorm = norm(g,Inf)
		if verbose
			@printf("%6d %15.5e %15.5e %15.5e\n",i,alpha,f,gradNorm)
		end

		# We want to stop if the gradient is really small
		if gradNorm < epsilon
			if verbose
				@printf("Problem solved up to optimality tolerance\n")
			end
			return w
		end
	end
	if verbose
		@printf("Reached maximum number of iterations\n")
	end
	return w
end

function findMinL1(funObj,w,lambda;maxIter=100,epsilon=1e-2)
	# funObj: function that returns (objective,gradient)
	# w: initial guess
	# lambda: value of L1-regularization parmaeter
	# maxIter: maximum number of iterations
	# epsilon: stop if the gradient gets below this

	# Evalluate the intial objective and gradient
	(f,g) = funObj(w)

	# Initial step size and sufficient decrease parameter
	gamma = 1e-4
	alpha = 1
	for i in 1:maxIter

		# Gradient step on smoooth part
		wNew = w - alpha*g
		# Proximal step on non-smooth part
		wNew = sign.(wNew).*max.(abs.(wNew) - lambda*alpha,0)
		(fNew,gNew) = funObj(wNew)

		# Decrease the step-size if we increased the function
		gtd = dot(g,wNew-w)
		while fNew + lambda*norm(wNew,1) > f + lambda*norm(w,1) - gamma*alpha*gtd
			@printf("Backtracking\n")
			alpha /= 2

			# Try out the smaller step-size
			wNew = w - alpha*g
			wNew = sign.(wNew).*max.(abs.(wNew) - lambda*alpha,0)
			(fNew,gNew) = funObj(wNew)
		end

		# Guess the step-size for the next iteration
		y = gNew - g
		alpha *= -dot(y,g)/dot(y,y)

		# Sanity check on the step-size
		if (!isfinitereal(alpha)) | (alpha < 1e-10) | (alpha > 1e10)
			alpha = 1
		end

		# Accept the new parameters/function/gradient
		w = wNew
		f = fNew
		g = gNew

		# Print out some diagnostics
		optCond = norm(w-sign.(w-g).*max.(abs.(w-g)-lambda,0),Inf)
		@printf("%6d %15.5e %15.5e %15.5e\n",i,alpha,f+lambda*norm(w,1),optCond)

		# We want to stop if the gradient is really small
		if optCond < epsilon
			@printf("Problem solved up to optimality tolerance\n")
			return w
		end
	end
	@printf("Reached maximum number of iterations\n")
	return w
end


import Pkg; Pkg.add(Pkg.PackageSpec(url="https://github.com/JuliaComputing/JuliaAcademyData.jl"))
using JuliaAcademyData; activate("Foundations of machine learning")

## Run this cell to load the graphics packages
using Plots; gr()

# ## Adding a function parameter

#-

# In the last notebook, we saw an example of adding **parameters** to functions. These are values that control the behavior of a function. Let's look at that in some more detail.

#-

# Let's go back to our original version of the σ function:

σ(x) = 1 / (1 + exp(-x))

# Instead of working with a single function, we can work with a whole class (set) of functions that look similar but differ in the value of a **parameter**. Let's make a new function that uses the previous $\sigma$ function, but also has a parameter, $w$. Mathematically, we could write
#
# $$f_w(x) = f(x; w) = \sigma(w \, x).$$
#
# (Here, $w$ and $x$ are multiplied in the argument of $\sigma$; we could write $w \times x$ or $w * x$, or even $w \cdot x$, but usually the symbols are not written.)
#
# Mathematically speaking, we can think of $f_w$ as a different function for each different value of the parameter $w$.

#-

# In Julia, we write this as follows:

f(x, w) = σ(w * x)

# Note that Julia just treats parameters as additional *arguments* to the function; the function `f` has two arguments, the value of `x` and the value of `w` that we want to use.

#-

# We can now investigate the effect of $w$ interactively. To do so, we need a way of writing in Julia "the function of one variable $x$ that we obtain when we fix the value of $w$". We write this as an "anonymous function", as we saw in the notebook on functions:
#
#     x -> f(x, w)
#
# We can read this as "the function that maps $x$ to the value of $f(x, w)$, for a value of $w$ that was previously given".

#-

# Now we are ready to draw the function. For each plot, we *fix* a value of the parameter $w$ and draw the resulting function as a function of $x$. We can interactively modify the value of $w$, and plot the new function that comes out:

## Try manipulating `w` between -2 and 2
w = 0.0
plot(x->f(x, w), -5, 5, ylims=(0,1), label="sigmoid")
plot!(x->(x>0), -5,5, label="Square Wave")

# #### Exercise 1
#
# Try writing your own function that takes a parameter. Start by copying and executing
#
# ```julia
# square(x) = x^2
# ```
#
# Then use `square` to declare a new function `square_and_scale` that takes two inputs, `a` and `x` such that
#
# $$\mathrm{square\_and\_scale}(x; a) := a \cdot x^2$$

#-

# #### Solution
#
# You should have declared the following function:
#
# ```julia
# square_and_scale(x, a) = a * x^2
# ```
#
# We can write some tests as follows, using the approximate equality operator, `≈`, written as `\approx<TAB>`:
#
# ```julia
# square_and_scale(1, 1) ≈ 1
# square_and_scale(1, 2) ≈ 2
# square_and_scale(2, 1) ≈ 4
# square_and_scale(2, 2) ≈ 8
# ```

#-

# #### Exercise 2

#-

# Once you have declared `square_and_scale`, uncomment the code below and see how the parameter `a` scales the function `square` :

## x = -10:10
## a = 5.0 # Try manipulating a here
## plot(x, square.(x), label="x^2")
## plot!(x, square_and_scale.(x, a), ls=:dash, label="ax^2")

# ## Fitting a function to data

#-

# As we saw in the previous notebook, what we would like to do is use the fact that we now have a parameter in our function in order to do something useful! Namely, we want to model data with it.

#-

# So suppose that we are given a single data point $(x_0, y_0) = (2, 0.8)$. We can try to "fit" the function $f_w$ by adjusting the parameter $w$ until the function passes through the data.
#
# **Game**: Change `w` until the graph of the function hits the data point. Which value of $w$ does that correspond to?

x0, y0 = 2, 0.8

w = 0.0 # try mainipulatoing `w` between -2 and 2
plot(x->f(x, w), -5, 5, ylims=(0, 1), label="f")
scatter!([x0], [y0], label="data")

# ## Quantifying how far we are from the goal: the *loss function*

#-

# We can see visually when the graph of the function passes through the data point. But the goal is to be able to automate this process so that the computer can do it unaided.
#
# So we will need a more precise way of deciding and quantifying (i.e. measuring with a number) *how far away we are from the goal*; here, the goal means hitting the data point with the function.

#-

# #### Exercise 3
#
# Can you think of a way of measuring how far away our function is from the data point?

#-

# ### Defining the loss function

#-

# We need to measure how far away the curve is from the data point when we choose a particular value of $w$.
# One way to do this is by finding the vertical distance $d$ from the curve to the data point.
#
# Instead of just taking the distance, it is common to take the *square* of the distance, $d^2$.
#
# Since we are taking the vertical distance, we need the distance at the given value of $x_0$ where the data point lies. For a given value of the parameter $w$, the height of the point on the curve with that value of $x_0$ is $f(x_0, w)$.
#
# So we take
# $$d := y_0 - f(x_0, w)$$
#
# and
# $$d^2 = [y_0 - f(x_0, w)]^2.$$
#
# This is our measure of distance. It will change when $w$ changes -- in other words, it is itself a *function of $w$*; we will denote this function by $L(w)$, and call it the **loss function**:
#
# $$L(w) := [y_0 - f(x_0, w)]^2.$$
#
# So the goal is to find the value $w^*$ of $w$ where the loss function is *least*; in other words, we need to *minimize* the loss function!
#
# (Another name for a loss function is a *cost function*.)

#-

# #### Exercise 4
#
# (a) Define the loss function `L(w)` in Julia.
#
# (b) Draw the data point and the function `x -> f(x, w)`. Also draw a vertical line from the data point to the function `x -> f(x, w)`.
#
# (c) Make the plot interactive.
#
# (d) Add as the plot title the value of the loss function for the current value of $w$.
#
# (e) Use the slider to find the value $w^*$ of $w$ for which the loss function reaches its minimum value. What is $w^*$? What is the value of the loss function there, $L(w^*)$?

#-

# #### Solution

#-

# (a) We have data

x0, y0 = 2, 0.8

# We define the loss function as follows:

L(w) = (y0 - f(x0, w))^2

# (b-d): The visualization can be written as follows

w = 0.0 # try manipulating w here between -2 and 2

plot(x->f(x, w), -5, 5, ylims=(0, 1), label="f_w", lw=3)  # function

scatter!([x0], [y0], label="data")
plot!([x0, x0], [y0, f(x0, w)], label="loss")  # vertical line

title!("L(w) = $(round(L(w); sigdigits = 5))")

# ## What does the loss function look like?

#-

# The loss function $L(w)$ tells us how far away the function $f_w$ is from the data when the parameter value is $w$, represented visually as the vertical line in the previous plot. When the data are fixed, this is a function only of the parameter $w$. What does this function look like as a function of $w$? Let's draw it!

#-

# #### Exercise 5
#
# Draw the function $L(w)$ as a function of $w$.

#-

# #### Solution

plot(L, -3, 3, xlabel="w", ylabel="L(w)", ylims=(0, 0.7), label="L")

# ### Features of the loss function

#-

# This graph quantifies how far we are from the data point for a given value of $w$.
# What features can we see from the graph?
#
# Firstly, we see that $L(w)$ is always bigger than $0$, for any value of $w$. This is because we want $L$ to be some kind of measure of *distance*, and distances cannot be negative.
#
# Secondly, we see that there is a special value $w^*$ of $w$ where the function $L$ reaches its minimum value. In this particular case, it actually reaches all the way down to $0$!
# This means that the original function $f$ (the one we manipulated above) passes exactly through the data point $(x_0, y_0)$.

#-

# #### Exercise 6
#
# Draw a zoomed-in version of the graph to find the place $w^*$ where the function hits $0$ more precisely.

#-

# ### A different way of defining the loss function

#-

# **Why did we use such a complicated function $L$ with those squares inside?** We could instead just have used the absolute distance, instead of the distance squared, using the *absolute value* function, written mathematically as $| \cdot |$, and in Julia as `abs`.

#-

# #### Exercise 7
#
# Define a new loss function, `L_abs`, using the absolute value, and see what it looks like.

#-

# #### Solution

L_abs(w) = abs(y0 - f(x0, w))

plot(L_abs, -3, 3, xlabel="w", ylabel="L_abs(w)", ylims=(0, 0.7), label="L_abs")

# Now we see why it was previously generally preferred to use squares: using the absolute value gives a cost function that is *not smooth*. This makes it difficult to use methods from calculus to find the minimum. Nonetheless, using non-smooth functions is very common in machine learning nowadays.

Courses/Foundations of machine learning/0500.Building-models-quiz.jl


# --- Imports

import Distributions
using Distributions: MvNormal, Uniform, ContinuousUnivariateDistribution, cdf, pdf
using HypothesisTests
using LinearAlgebra
using Plots
using Random

# --- Generate sample of vectors drawn from a uniform distribution on a unit circle

n = 2
num_samples_2d = 50000

dist = Uniform(-π, π)
num_vectors = num_samples_2d
theta = rand(dist, num_vectors)
vectors = transpose(hcat(cos.(theta), sin.(theta)));

# --- Plot histogram of angles

thetas = map(i -> atan(vectors[:, i][2], vectors[:, i][1]), 1:num_vectors)

num_hist_bins = 25
hist_bins = range(-π, π; length=num_hist_bins)
hist = histogram(thetas; bins=hist_bins, normalize=true)
plt = plot(hist)

pdf_2d = 0.5 / π
println("pdf(θ): $(pdf_2d)")

# Display plot
display(plt)

# --- Perform test for uniformity on the circle

# Analytical formula for distribution over angle from x-axis
struct ThetaDistribution2D <: ContinuousUnivariateDistribution end
Distributions.pdf(dist::ThetaDistribution2D, x::Real) = 0.5 / π
Distributions.cdf(dist::ThetaDistribution2D, x::Real) = 0.5 * (x + π) / π

theta_dist = ThetaDistribution2D()

# Perform Anderson-Darling Test
test_results = OneSampleADTest(thetas, theta_dist)
println(test_results)

test_results = ExactOneSampleKSTest(thetas, theta_dist)
println(test_results)

test_results = ApproximateOneSampleKSTest(thetas, theta_dist)
println(test_results)

# --- Generate sample of vectors drawn from a uniform distribution on a unit circle

n = 2
num_samples_2d = 50000

# Generate vectors
dist = MvNormal(zeros(n), ones(n))
num_vectors = num_samples_2d
vectors = rand(dist, num_vectors)
for i = 1:num_vectors
    vectors[:, i] /= norm(vectors[:, i])
end

# --- Plot histogram of angles

thetas = map(i -> atan(norm(vectors[:, i][2]), vectors[:, i][1]), 1:num_vectors)

num_hist_bins = 25
hist_bins = range(0, π; length=num_hist_bins)
hist = histogram(thetas; bins=hist_bins, normalize=true)
plt = plot(hist)

pdf_2d = 1 / π
println("pdf(θ): $(pdf_2d)")

# Display plot
display(plt)

# --- Perform test for uniformity on the circle

# Analytical formula for distribution over angle from x-axis
struct ThetaDistribution2D <: ContinuousUnivariateDistribution end
Distributions.pdf(dist::ThetaDistribution2D, x::Real) = 1 / π
Distributions.cdf(dist::ThetaDistribution2D, x::Real) = x / π

theta_dist = ThetaDistribution2D()

# Perform Anderson-Darling Test
test_results = OneSampleADTest(thetas, theta_dist)
println(test_results)

test_results = ExactOneSampleKSTest(thetas, theta_dist)
println(test_results)

test_results = ApproximateOneSampleKSTest(thetas, theta_dist)
println(test_results)

# --- Generate sample of vectors drawn from a uniform distribution on a unit sphere

n = 3
num_samples_3d = 50000

# Generate vectors
dist = MvNormal(zeros(n), ones(n))
num_vectors = num_samples_3d
vectors = rand(dist, num_vectors)
for i = 1:num_vectors
    vectors[:, i] /= norm(vectors[:, i])
end

# --- Plot histogram of angles

thetas = map(i -> atan(norm(vectors[:, i][2:end]), vectors[:, i][1]), 1:num_vectors)

num_hist_bins = 50
hist_bins = range(0, π; length=num_hist_bins)
hist = histogram(thetas; bins=hist_bins, normalize=true)
plt = plot(hist)

pdf_3d(x) = 0.5 * sin(x)
plt = plot!(pdf_3d, 0, π)

# Display plot
display(plt)

# --- Perform test for uniformity on the sphere

# Analytical formula for distribution over angle from x-axis
struct ThetaDistribution3D <: ContinuousUnivariateDistribution end
Distributions.cdf(dist::ThetaDistribution3D, x::Real) = 0.5 * (1 - cos(x))

theta_dist = ThetaDistribution3D()

# Perform Anderson-Darling Test
test_results = OneSampleADTest(thetas, theta_dist)
println(test_results)

test_results = ExactOneSampleKSTest(thetas, theta_dist)
println(test_results)

test_results = ApproximateOneSampleKSTest(thetas, theta_dist)
println(test_results)

# --- Generate sample of vectors drawn from a uniform distribution on a unit sphere

n = 20
num_samples = 50000

# Generate vectors
dist = MvNormal(zeros(n), ones(n))
num_vectors = num_samples
vectors = rand(dist, num_vectors)
for i = 1:num_vectors
    vectors[:, i] /= norm(vectors[:, i])
end

# --- Plot histogram of angles

thetas = map(i -> atan(norm(vectors[:, i][2:end]), vectors[:, i][1]), 1:num_vectors)

num_hist_bins = 50
hist_bins = range(0, π; length=num_hist_bins)
hist = histogram(thetas; bins=hist_bins, normalize=true)
plt = plot(hist)

pdf_nd(x) = (2^(n-2) / π / binomial(n-2, (n-2) ÷ 2)) * sin(x)^(n-2)
plt = plot!(pdf_nd, 0, π)

# Display plot
display(plt)

# --- Perform test for uniformity on the hypersphere

# Analytical formula for distribution over angle from x-axis
struct ThetaDistributionND <: ContinuousUnivariateDistribution end

function Distributions.cdf(dist::ThetaDistributionND, x::Real)
    k = n-2
    sgn = (-1)^(k÷2)
    value = 0
    for j = 0:(k÷2 - 1)
        value += (-1)^j * binomial(k, j) * sin((k - 2*j) * x) / (k - 2*j)
    end
    value *= 2
    value += sgn * binomial(k, k÷2) * x  # contribution from i = k÷2 term
    value *= sgn / π / binomial(k, k÷2)

    return value
end

theta_dist = ThetaDistributionND()

# Perform Anderson-Darling Test
test_results = OneSampleADTest(thetas, theta_dist)
println(test_results)

test_results = ExactOneSampleKSTest(thetas, theta_dist)
println(test_results)

test_results = ApproximateOneSampleKSTest(thetas, theta_dist)
println(test_results)

# --- Generate sample of vectors drawn from a uniform distribution on a unit sphere

n = 45
num_samples = 50000

# Generate vectors
dist = MvNormal(zeros(n), ones(n))
num_vectors = num_samples
vectors = rand(dist, num_vectors)
for i = 1:num_vectors
    vectors[:, i] /= norm(vectors[:, i])
end

# --- Plot histogram of angles

thetas = map(i -> atan(norm(vectors[:, i][2:end]), vectors[:, i][1]), 1:num_vectors)

num_hist_bins = 50
hist_bins = range(0, π; length=num_hist_bins)
hist = histogram(thetas; bins=hist_bins, normalize=true)
plt = plot(hist)

pdf_nd(x) = (2^(n-2) / π / binomial(n-2, (n-2) ÷ 2)) * sin(x)^(n-2)
plt = plot!(pdf_nd, 0, π)

# Display plot
display(plt)

# --- Perform test for uniformity on the hypersphere

# Analytical formula for distribution over angle from x-axis
struct ThetaDistributionND <: ContinuousUnivariateDistribution end

function Distributions.cdf(dist::ThetaDistributionND, x::Real)
    k = n-2
    value = 0
    norm_ = 0
    for j = 0:(k-1)÷2
        coef = (-1)^j * binomial(k, j) / (k - 2*j)
        value += coef * (1 - cos((k - 2*j) * x))
        norm_ += coef
    end
    value /= 2 * norm_

    return value
end

theta_dist = ThetaDistributionND()

# Perform Anderson-Darling Test
test_results = OneSampleADTest(thetas, theta_dist)
println(test_results)

test_results = ExactOneSampleKSTest(thetas, theta_dist)
println(test_results)

test_results = ApproximateOneSampleKSTest(thetas, theta_dist)
println(test_results)


using Statistics, StatsBase, LinearAlgebra, DataFrames, CSV
df = CSV.read("../data/3featureData.csv",header=false)
n, p = size(df)
println("Number of features: ", p)
println("Number of observations: ", n)
X = convert(Array{Float64,2},df)
println("Dimensions of data matrix: ", size(X))

xbarA = (1/n)*X'*ones(n)
xbarB = [mean(X[:,i]) for i in 1:p]
xbarC = sum(X,dims=1)/n
println("\nAlternative calculations of (sample) mean vector: ")
@show(xbarA), @show(xbarB), @show(xbarC)

Y = (I-ones(n,n)/n)*X
println("Y is the de-meaned data: ", mean(Y,dims=1))

covA = (X .- xbarA')'*(X .- xbarA')/(n-1)
covB = Y'*Y/(n-1)
covC = [cov(X[:,i], X[:,j]) for i in 1:p, j in 1:p]
covD = [cor(X[:,i], X[:,j])*std(X[:,i])*std(X[:,j]) for i in 1:p, j in 1:p]
covE = cov(X)
println("\nAlternative calculations of (sample) covariance matrix: ")
@show(covA), @show(covB), @show(covC), @show(covD), @show(covE)

ZmatA = [(X[i,j] - mean(X[:,j]))/std(X[:,j]) for i in 1:n, j in 1:p ]
ZmatB = hcat([zscore(X[:,j]) for j in 1:p]...)
println("\nAlternate computation of Z-scores yields same matrix: ",
	maximum(norm(ZmatA-ZmatB)))
Z = ZmatA

corA = covA ./ [std(X[:,i])*std(X[:,j]) for i in 1:p, j in 1:p]
corB = covA ./ (std(X,dims = 1)'*std(X,dims = 1))
corC = [cor(X[:,i],X[:,j]) for i in 1:p, j in 1:p]
corD = Z'*Z/(n-1)
corE = cov(Z)
corF = cor(X)
println("\nAlternative calculations of (sample) correlation matrix: ")
@show(corA), @show(corB), @show(corC), @show(corD), @show(corE), @show(corF);


function parabolicinterp(f::Function, xl::Number, xm::Number, xu::Number, ea::Number, maxIter::Int64)
  # 1D optimisation using parabolic interpolation

  xo = NaN; # Current estimate of the optimum
  fxm = f(xm);
  fxu = f(xu);
  fxl = f(xl);

  for i = 1:maxIter
    xoold = xo; # xoxo

    # Solution for xo using simultaneous equations
    fracNumerator = ((xm - xl)^2 * (fxm-fxu)) - ((xm - xu)^2 * (fxm - fxl));
    fracDenominator = ((xm - xl) * (fxm - fxu)) - ((xm - xu) * (fxm - fxl));
    xo = xm - 0.5 * (fracNumerator / fracDenominator);

    if xo < xm
      # The optimum is to the left of xm so make the upper estimate xm and make
      # the middle estimate xo.
      xu = xm;
      fxu = fxm;
      xm = xo;
      fxm = f(xm);
    elseif xo > xm
      # The optimum is to the right of xm so make the lower estimate xm and
      # the middle estimate xo.
      xl = xm;
      fxl = fxm;
      xm = xo;
      fxm = f(xm);
    else
      break;
    end

    # Relative error checks
    if abs((xo - xoold) / xo) * 100 < ea
      break;
    end
  end
  return xo;
end


module YaLinearAlgebra

import Base: +, -, *

export +, -, *, μ, dot, sum_of_square, norm,
  distance, shape, get_row, get_column, get_col, make_matrix

## Vector Portion

function +(v₁::Vector{T}, v₂::Vector{T})::Vector{T} where T <: Number
  """
  Vector Additon is component-wise
  """
  @assert length(v₁) == length(v₂)
  v₁ .+ v₂
end

function +(vc::Vector{Vector{T}})::Vector{T} where T <: Number
  """
  Sum all corresponding components of a collection of vector to return
  one vector
  """
  @assert length(vc) > 0 "expect the collection to have at leat one vector element"
  n = length(vc[1])
  @assert all(v -> length(v) == n, vc) "Expect all vectors to be of he same length"
  v = zeros(T, n)
  for v₀ ∈ vc
    v .+= v₀
  end
  v
end

function -(v₁::Vector{T}, v₂::Vector{T})::Vector{T} where T <: Number
  """
  Vector Additon is component-wise
  """
  @assert length(v₁) == length(v₂)
  v₁ .- v₂
end

*(x::T, v::Vector{T}) where T <: Number = v .* x
*(v::Vector{T}, x::T) where T <: Number = v .* x

function μ(vc::Vector{Vector{T}})::Vector{AbstractFloat} where T <: Number
  """Element-wise mean"""
  @assert length(vc) > 0
  n = length(vc[1])

  @assert all(v -> length(v) == n, vc)
  reduce(+, vc) / length(vc)
end

function dot(v₁::Vector{T}, v₂::Vector{T})::T where T <: Number
  @assert length(v₁) == length(v₂)
  sum(v₁ .* v₂)
end

function sum_of_square(v::Vector{T})::T where T <: Number
  """
  return vᵢ * vᵢ ∀ i ∈ 1:length(v)
  """
  dot(v, v)
end

function norm(v::Vector{T})::AbstractFloat where T <: Number
  √ sum(v .* v)
end

function distance(v₁::Vector{T}, v₂::Vector{T})::AbstractFloat where T <: Number
  """
  (Euclidean) distance(v₁, v₂) ≡ √(Σ (v₁ᵢ - v₂ᵢ)²)
  """
  @assert length(v₁) == length(v₂)
  √ sum((v₁ .- v₂).^2)  # == norm(v₁ .- v₂)
end

## Matrix Portion

function shape(m::Matrix{T})::Tuple{Integer, Integer} where T <: Number
  size(m)
end

function get_row(m::Matrix{T}, r::Integer)::Vector{T}  where T <: Number
  @assert 1 ≤ r ≤ size(m)[1]
  view(m, r, :)  ## no copy!
end

function get_column(m::Matrix{T}, c::Integer)::Vector{T}  where T <: Number
  @assert 1 ≤ c ≤ size(m)[2]
  view(m, :, c)  ## no copy!
end

get_col(m::Matrix{T}, c::Integer) where T <: Number = get_column(m, c)

function make_matrix(nrows::Integer, ncols::Integer, fn::Function;
    DT::DataType=Float64)::Matrix
  @assert 1 ≤ nrows && 1 ≤ ncols

  m = zeros(DT, (nrows, ncols))
  for c ∈ 1:ncols, r ∈ 1:nrows
    m[r, c] = fn(r, c)
  end
  m
end

end  ## YaLinearAlgebra

Data Science From Scratch/src/YaLinearAlgebra.jl


using LinearAlgebra
using SparseArrays
using Plots
using ForwardDiff

"This code performs a convergence test for two finite difference methods applied to
Laplace's equation with mixed Dirichlet-Neumann boundary conditions."

# method of manufactured solutions: define the exact solution, then figure out
# what f,α,β should be to make it the solution to your PDE
uexact(x) = log(2+sin(pi*x))
dudx_exact(x) = ForwardDiff.derivative(uexact,x)
f(x) = -ForwardDiff.derivative(dudx_exact,x) # -d^2u/dx^2 = f

α = uexact(-1)
β = dudx_exact(1)

# The O(h) accurate method
function solve1(m,f,α,β)
    x = LinRange(-1,1,m+2) # add 2 endpoints
    xint = x[2:end]
    h = x[2]-x[1]
    A = (1/h^2) * spdiagm(0=>2*ones(m+1),-1=>-ones(m),1=>-ones(m))
    A[end,end] = 1/h^2
    b = f.(xint)
    b[1] += α/h^2

    b[end] += β/h # approach 1 - O(h) accuracy only

    u = A\b
    return u,x,xint,h
end

# The O(h^2) accurate method
function solve2(m,f,α,β)
    x = LinRange(-1,1,m+2) # add 2 endpoints
    xint = x[2:end] # leave x_{m+1} in the list of points!
    h = x[2]-x[1]
    A = (1/h^2) * spdiagm(0=>2*ones(m+1),-1=>-ones(m),1=>-ones(m))
    A[end,end] = 1/h^2
    b = f.(xint)
    b[1] += α/h^2

    b[end] = .5*f(x[end]) + β/h

    u = A\b
    return u,x,xint,h
end

# # this code plots the solution
# m = 40
# u1,x,xint1,h = solve1(m,f,α,β)
# u2,x,xint2,h = solve2(m,f,α,β)
# plot(xint1,u1,mark=:dot,markersize=2,label="O(h) solution")
# plot!(xint2,u2,mark=:dot,markersize=2,label="O(h^2) solution")
# plot!(x,uexact.(x),label="Exact solution")
# plot!(leg=:bottomright)

mvec = 2 .^ (2:9)
hvec = zeros(length(mvec))
err1 = zeros(length(mvec))
err2 = zeros(length(mvec))
for (i,m) in enumerate(mvec)
    u1,x,xint,h = solve1(m,f,α,β)
    u2,x,xint,h = solve2(m,f,α,β)
    hvec[i] = h
    err1[i] = maximum(@. abs(uexact(xint) - u1))
    err2[i] = maximum(@. abs(uexact(xint) - u2))
end
plot(hvec,err1,marker=:dot,label="Max error for method 1")
plot!(hvec,err2,marker=:dot,label="Max error for method 2")
plot!(hvec,hvec.^2,linestyle=:dash,label="O(h^2)")
plot!(xaxis=:log,yaxis=:log,legend=:bottomright)


using LinearAlgebra

# k_local(Elastic Modulus,thickness,height,length,number of nodes,row/column one, row/column two)

#creates local stiffness matrix and maps to global
function k_local(k,l,n,rc_1,rc_2)
    k_global = zeros(n,n)
    k_global[rc_1,rc_1] = k
    k_global[rc_1,rc_2] = -k
    k_global[rc_2,rc_1] = -k
    k_global[rc_2,rc_2] = k
    return k_global
end

# constants from problem statement
E = 29*10^6
l = 2
k1 = 25
k2 = 35
k3 = 45
k4 = 30
# combining all globally mapped matrices
kG = k_local(k1,l,5,1,2)+
     k_local(k2,l,5,2,3)+
     k_local(k3,l,5,3,4)+
     k_local(k4,l,5,3,5)
kG[4,:] = Float64[0 0 0 1 0] #adjustment for Reaction from fixed end
kG[5,:] = Float64[0 0 0 0 1] #adjustment for Reaction from fixed end
kG[:,4] = transpose(Float64[0 0 0 1 0]) #adjustment for Reaction from fixed end
kG[:,5] = transpose(Float64[0 0 0 0 1]) #adjustment for Reaction from fixed end

# Force Vector
f = Float64[75 0 0 0 0]
F = transpose(f)

# Displacement vector
u = kG\F
println("Nodal displacement vector (inches):")
display(u)

# Strain Function
ϵ(x2,x1,l) = (x2 - x1)/l

#Strain Vector
ϵM = [ϵ(u[2],u[1],l)
      ϵ(u[3],u[2],l)
      ϵ(u[4],u[3],l)
      ϵ(u[5],u[3],l)]
println("Element strain vector (inch/inch):")
display(ϵM)

# Stress Function
σ(E,ϵ) = E*ϵ

# Stress Vector
σM = [σ(E,ϵM[1])
      σ(E,ϵM[2])
      σ(E,ϵM[3])
      σ(E,ϵM[4])]
println("Element Stress vector (psi):")
display(σM)

#Reaction Forces vector
Fapp(k,x2,x) = k*(x2-x)
FappM = [Fapp(k12,u[2],u[1])
         Fapp(k12,u[3],u[2])
         Fapp(k3,u[4],u[3])
         Fapp(k4,u[5],u[3])]
println("Reaction Forces vector (lbs):")
display(FappM)


"""
    univariate(k::Int64, l::Int64, low::N, high::N)::Vector{N} where {N<:AbstractFloat}

Compute the Bernstein coefficients of a univariate monomial.

### Input

- `k`    -- degree of the given monomial
- `l`    -- degree of the Bernstein polynomial
- `low`  -- the lower bound of the interval where the Bernstein coefficients are computed
- `high` -- the upper bound of the interval the Bernstein coefficients are computed

### Output

A vector with floating point entries containing the Bernstein coefficients.
"""
function univariate(k::Int64, l::Int64, low::N, high::N)::Vector{N} where {N<:AbstractFloat}
    m = l - k
    b = zeros(N, l+1)
    @inbounds @fastmath for i in 0:l
        if k < l
            for j in max(0, i-m):min(k, i)
                aux =  binomial(m, i-j) * binomial(k, j) / binomial(k+m, i)
                b[i+1] += aux * low^(k-j) * high^j
            end
        else
            b[i+1] = low^(k-i) * high^i
        end
    end
    return b
end

"""
    univariate(k::Int64, l::Int64, low::Rational, high::Rational)::Vector{Rational}

Compute *exactly* the Bernstein coefficients of a univariate monomial.

### Input

- `k`    -- degree of the given monomial
- `l`    -- degree of the Bernstein polynomial
- `low`  -- the lower bound of the interval where the Bernstein coefficients are computed
- `high` -- the upper bound of the interval the Bernstein coefficients are computed

### Output

A vector with rational entries containing the Bernstein coefficients.
"""
function univariate(k::Int64, l::Int64, low::Rational, high::Rational)::Vector{Rational}
    m = l - k
    b = zeros(Rational, l+1)
    @inbounds for i in 0:l
        if k < l
            for j in max(0, i-m):min(k, i)
                aux =  binomial(m, i-j) * binomial(k, j) // binomial(k+m, i)
                b[i+1] += aux * low^(k-j) * high^j
            end
        else
            b[i+1] = low^(k-i) * high^i
        end
    end
    return b
end

"""
    multivariate(k::Vector{Int64}, l::Vector{Int64},
                          low::Vector{N}, high::Vector{N})::ImplicitForm{N} where {N<:Number}

Compute the Bernstein coefficients of a multivariate monomial.

### Input

- `k`    -- vector of degrees for each monomial
- `l`    -- vector of degrees of the Bernstein polynomial for each monomial
- `low`  -- the lower bounds of the interval where the Bernstein coefficients are computed
- `high` -- the upper bounds of the interval the Bernstein coefficients are computed

### Output

A Bernstein implicit form holding the Bernstein coefficients.

### Examples

```julia
julia> m = multivariate([3,2],[3,2],[1.0,2],[2,4.0]);
julia> m.array
2-element Array{Array{Float64,1},1}:
 [1.0, 2.0, 4.0, 8.0]
 [4.0, 8.0, 16.0]
```
"""
function multivariate(k::Vector{Int64}, l::Vector{Int64},
                      low::Vector{N}, high::Vector{N})::ImplicitForm{N} where {N<:Number}
    n = length(low)
    ncoeffs = 1
    B = Vector{Vector{N}}(n)
    @inbounds for i in 1:n
        @views B[i] = univariate(k[i], l[i], low[i], high[i])
        ncoeffs = ncoeffs * length(B[i])
    end
    return ImplicitForm(B, n, ncoeffs)
end


using ForwardDiff, Roots, Plots, LinearAlgebra, Optim

function strong_backtracking(f, ∇, x, d; α=1, β=0.1, σ=0.4)
    y0, g0, y_prev, α_prev = f(x), ∇(x)⋅d, NaN, 0
    αlo, αhi = NaN, NaN
    # bracket phase
    while true
        y = f(x + α*d)
        if y > y0 + β*α*g0 || (!isnan(y_prev) && y ≥ y_prev)
            αlo, αhi = α_prev, α
            break
        end
        g = ∇(x + α*d)⋅d
        if abs(g) ≤ -σ*g0
            return α
        elseif g ≥ 0
            αlo, αhi = α, α_prev
            break
        end
        y_prev, α_prev, α = y, α, 2α
    end

    # zoom phase
    ylo = f(x + αlo*d)
    while true
        α = (αlo + αhi)/2
        y = f(x + α*d)
        if y > y0 + β*α*g0 || y ≥ ylo
            αhi = α
        else
            g = ∇(x + α*d)⋅d
            if abs(g) ≤ -σ*g0
                return α
            elseif g*(αhi - αlo) ≥ 0
                αhi = αlo
            end
            αlo = α
        end
    end
end

f(x) = ((x[1]-1.0)^2) + (0.1*(x[2]-4)^2) + sin(5*pi*x[1])+sin(5*pi*x[2])
grad_f(x) = ForwardDiff.gradient(f,x)

# Ponto inicial
xk = [-1.0,1.0]

#direcao de busca
dk = -grad_f(xk)/norm(grad_f(xk))

beta = 0.1
alpha_f = 1.0

phi(alpha) = f(xk+alpha*dk)
d_phi(alpha) = grad_f(xk+alpha*dk)'dk

l_alpha(alpha) = phi(0)+alpha*beta*d_phi(0)
sigma = 0.4

wolf = []

for alpha = 0:0.01:alpha_f
    if (phi(alpha) <= l_alpha(alpha)) && (abs(grad_f(xk+alpha*dk)'dk) <= -sigma*grad_f(xk)'dk)
        push!(wolf,alpha)
    end
end

wolfbt = strong_backtracking(f,grad_f,xk,dk)

plot(phi,0,alpha_f,lw=2,lab="phi")
plot!(l_alpha,0,alpha_f,lw=2,lab="L_phi",c=:black)
scatter!(wolf,phi.(wolf),lab="StrongWolfe",ms=3,c=:red)
scatter!([wolfbt],[phi(wolfbt)],lab="StrongWolf - BT",c=:yellow)
plot!(legend=:outerbottomright)
plot!(title="beta = 0.1 e sigma = 0.4",xlabel = "alpha", ylabel = "phi(alpha), l_alpha(alpha")


using SpecialMatrices
using LinearAlgebra
using SparseArrays

"""Ankara.jl contain the functions to develop the description of the
discrete and finite harmonic oscillator in terms of the Harper equations."""

# Δ1(j) gives the circulant matrix of size d_{j} × d_{j} where d_{j} = 2j + 1
function Δ1(j)
	Δ1 = zeros(Float64, 2j + 1)
	Δ1[1] = -2
	Δ1[2] = 1
	Δ1[end] = 1
	Δ1 = SpecialMatrices.Circulant(Δ1)
end

# Δ2(j) gives the diagonal matrix of size d_{j} × d_{j} whose diagonal entries
# are oscillating terms
function Δ2(j)
	Δ2 = zeros(Float64, 2j + 1)
	for i in -j:j
		Δ2[i + 1 + j] = -4 * (sin(pi * i / (2j)))^2
	end
	return Δ2 = LinearAlgebra.diagm(Δ2)
end

# If necessary, compose the Hamiltonian (Harper's equation) H from who the
# eigenvectors and eigenvalues are obtained
H(j) = -(1/2.0)*(Δ1(j) + Δ2(j))

# Eigenvectors of H
h(j) = LinearAlgebra.eigvecs(H(j));

# Eigenvalues of H. Altough, this eigvenvalues are not strictly used because
# of the symmetry importation below
k(j) = LinearAlgebra.eigvals(H(j));

# Lineal monotonic increasing values who serves as the (new) eigenvalues
# in the symmetry importation to construct the evolution operator
η(j) = [l for l in 0:(2j + 1)];

# Kernel of the evolution
ker(j, t) = exp.(-1im * pi * t * η(j)/2.0)


#' ## Run this cell to load the graphics packages
using Plots
gr()

#' ## Run this cell to use a couple of definitions from the previous
#' Tools notebook

sigma(x) = 1 / (1 + exp(-x))
f(x, w) = sigma(w * x)

#' ## Multiple function parameters
#'
#' In notebook 6, we saw how we could adjust a parameter to make a curve fit
#' a single data point. What if there is more data to fit?
#'
#' We'll see that as we acquire more data to fit, it can sometimes be useful
#' to add complexity to our model via additional parameters.

#' ## Multiple function parameters
#'
#' In notebook 6, we saw how we could adjust a parameter to make a curve fit
#' a single data point. What if there is more data to fit?
#'
#' We'll see that as we acquire more data to fit, it can sometimes be useful
#' to add complexity to our model via additional parameters.

#' ## Adding more data
#' Suppose there are now two data points to fit, the previous $(x_0, y_0)$
#' used in notebook 6 and also $(x_1, y_1) = (-3, 0.3)$.

#' #### Exercise 1
#
# Make an interactive plot of the function $f_w$ together with the two data
# points. Can you make the graph of $f_w$ pass through *both* data points
# at the *same* time?

x0, y0 = 2, 0.8
x1, y1 = -3, 0.3
w = 0.4
plot(x->f(x, w), -5, 5, ylims=(0, 1), label="f")
scatter!([x0,x1], [y0,y1], label="data")

#' You should have found that it's actually *impossible* to fit both data points
#' at the same time! The best we could do is to *minimise* how far away the
#' function is from the data. To do so, we need to somehow balance the distance
#' from each of the two data points.

#' #### Exercise 2
#'
#' Play with the slider to find the value $w^*$ of $w$ that you think has the "least error" in an intuitive sense.

w=0.45
plot(x->f(x, w), -5, 5, ylim=(0,1))
scatter!([x0, x1], [y0, y1])

#' ## Defining a loss function
#' How can we *quantify* some kind of overall measure of the distance from *all* of the data? Well, we just need to define a new loss function! One way to do so would be to sum up the loss functions for each data point, i.e. the sum of the squared vertical distances from the graph to each data point:
#'
#' $$L(w) = [y_0 - f_w(x_0)]^2 + [y_1 - f_w(x_1)]^2.$$
#'
#' Since the two pieces that we are adding have the same mathematical "shape" or structure, we can abbreviate this by writing it as a sum:
#'
#' $$L(w) = \sum_{i=0}^1 [y_i - f_w(x_i)]^2.$$

#' So now we want to find the value $w^*$ of $w$ that minimizes this new
#' function $L$!

#' #### Exercise 3
#'
#' Make a visualization to show the function $f_w$ and to visualize the
#' distance from each of the data points.
xs = [2, -3]
ys = [0.8, 0.3]

L(w) = sum( (ys .- f.(xs, w)) .^ 2 )

begin
    w = 0.3 # Try manipulating w between -2 and 2

    plot(x->f(x, w), -5, 5, ylims=(0, 1), lw = 3, label="f_w")

    scatter!(xs, ys, label="data")

    for i in 1:2
        plot!([xs[i], xs[i]], [ys[i], f(xs[i], w)], c=:green)
    end

    title!("L(w) =  $(round(L(w); sigdigits = 5))")
end

#' #### Exercise 4
#'
#' After playing with this for a while, it is intuitively obvious that we
# cannot make the function pass through both data points for any value of
# $w$. In other words, our loss function, `L(w)`, is never zero.
#'
#' What is the minimum value of `L(w)` that you can find by altering `w`?
#' What is the corresponding value of `w`?
w = 0.42
plot(x->f(x, w), -5, 5, ylims=(0, 1), lw = 3, label="f_w", legend=(0.8,0.1))

scatter!(xs, ys, label="data")

for i in 1:2
    plot!([xs[i], xs[i]], [ys[i], f(xs[i], w)], c=:green)
end

title!("L($w) =  $(round(L(w); sigdigits = 5))")

#' ### Sums in Julia
#' To generate the above plot we used the `sum` function.
#' `sum` can add together all the elements of a collection or range,
#' or it can add together the outputs of applying a function to all the
#' elements of a collection or range.
#'
#' Look up the docs for `sum` via
#'
#' ```julia
#' ?sum
#' ```
#' if you need more information.

#' #### Exercise 6
#'
#' What is the sum of the absolute values of all integers between -3 and 3? Use `sum` and the `abs` function.

#' #### Solution

sum(abs, -3:3)

#' ## What does the loss function $L$ look like?
#' In our last attempt to minimize `L(w)` by varying `w`, we saw that `L(w)`
#' always seemed to be greater than 0.  Is that true? Let's plot it to find out!

#' #### Exercise 7
#'
#' Plot the new loss function $L(w)$ as a function of $w$.
#' #### Solution
plot(L, -2, 2, xlabel="w", ylabel="L(w)", ylims=(0, 1.2))

#' ### Features of the loss function
#' The first thing to notice is that $L$ is always positive.
#' Since it is the sum of squares, and squares cannot be negative,
#' the sum cannot be negative either!
#'
#' However, we also see that although $L$ dips close to $0$ for a single,
#' special value $w^* \simeq 0.4$, it never actually *reaches* 0!
#' Again we could zoom in on that region of the graph to estimate it more
#' precisely.

#' We might start suspecting that there should be a better way of using the
#' computer to minimize $L$ to find the location $w^*$ of the minimum,
#' rather than still doing everything by eye. Indeed there is, as we will
#' see in the next two notebooks!

#' ## Adding more parameters to the model

#' If we add more parameters to a function, we may be able to improve how it
#' fits to data. For example, we could define a new function $g$ with another
#' parameter, a shift or **bias**:
#'
#' $$g(x; w, b) := \sigma(w \, x + b).$$
#+ results="hidden"
g(x, w, b) = sigma(w*x + b)
#+

xs = [2, -3]
ys = [0.8, 0.3]

L2D(w, b) = sum( (ys .- g.(xs, w, b)) .^ 2)
w = 0.45 # Try manipulating w between -2 and 2
b = 0.5 # Try manipulating b between -2 and 2
plot(x -> g(x, w, b), -5, 5, ylims=(0, 1), lw=3)

scatter!(xs, ys)

for i in 1:2
    plot!([xs[i], xs[i]], [ys[i], g(xs[i], w, b)])
end

title!("L2D(w, b) = $(L2D(w, b))")

#' You should be able to convince yourself that we can now make the curve pass
#' through both points simultaneously.

#' #### Exercise 9
#'
#' For what values of `w` and `b` does the line pass through both points?

w = 0.45
b = 0.48
plot(x -> g(x, w, b), -5, 5, ylims=(0, 1), lw=3)

scatter!(xs, ys)

for i in 1:2
    plot!([xs[i], xs[i]], [ys[i], g(xs[i], w, b)])
end

title!("L2D(w, b) = $(L2D(w, b))")
#' ## Fitting both data points: a loss function
#' Following the procedure that we used when we had a single parameter,
#' we can think of the fitting procedure once again as minimizing a suitable
#' *loss function*. The expression for the loss function is almost the same,
#' except that now $f$ has two parameters $w$ and $b$, so the loss function
#' $L_2$ is itself a function of $w$ *and* $b$:
#'
#' $$L_2(w, b) = \sum_i [y_i - f(x_i; w, b)]^2.$$
#'
#' So we want to minimize this loss function over *both* of the parameters
#' $w$ and $b$! Let's plot it.

#' #### Exercise 10
#'
#' Define the function `L2` in Julia.

#' #### Solution

L2(w, b) = sum( (ys .- g.(xs, w, b)) .^ 2 )

#' To plot the loss as a function of *two* variables (the weights *and* the bias),
#' we will make use of the `surface` function.  To get a nice interactive
# 3D plot, we will use the PlotlyJS "backend" (plotting engine):

gr()  # load the backend

#-

ws = -2:0.05:2
bs = -2:0.05:2

contour(ws, bs, L2, levels=0.001:0.025:1.25, xlabel="value of w", ylabel="value of b")

#' We can see that indeed there is a unique point $(w^*, b^*)$ where the
#' function $L_2$ attains its minimum. You can try different ranges for the
#' $x$, $y$ and $z$ coordinates to get a better view.

#' ## More data

#' If we add more data, however, we will again not be able to fit all of the
#' data; we will only be able to attain a "best fit".
#'
#' Let's create `xs` and `ys` with some more data:

xs = [2, -3, -1, 1]
ys = [0.8, 0.3, 0.4, 0.4]

#' #### Exercise 11
#'
#' a) Make an interactive visualization of the function $f$ and the data. Try
#' to find the values of $w$ and $b$ that give the best fit.
#'
#' b) Define the loss function and plot it.
#'
#' ```julia
#' sigma(x) = 1 / (1 + exp(-x))
#' f(x, w) = sigma(w * x)
#' g(x, w, b) = sigma(w*x + b)
#' ```
L2(w, b) = sum( (ys .- g.(xs, w, b)) .^ 2 )

w = 0.4 # Try manipulating w between -2 and 2
b = -0.1 # Try manipulating b between -2 and 2
plot(x->g(x, w, b), -5, 5, ylims=(-0.2, 1))

scatter!(xs, ys)

for i in 1:length(xs)
    plot!([xs[i], xs[i]], [ys[i], g(xs[i], w, b)])
end

title!("L2(w, b) = $(round(L2(w, b); sigdigits = 5))")

#' Let's define the loss function that we're using above so that we can plot
#' it as a function of the parameters `w` and `b`.
#' #### Exercise 12
#'
#' We've seen the loss function, $$L_{2D}(w, b) = \sum_i[\mathrm{ys}_i - g(\mathrm{xs}_i, w, b)]^2,$$ written in Julia as
#
# ```julia
# L2(w, b) = sum( (ys .- g.(xs, w, b)) .^ 2 )
# ```
#
# a few times now. To ensure you understand what this function is doing, implement your own loss function using the commented code below. Do this without using `sum`, `abs2`, or broadcasting dot syntax (for example, `.-`). Hint: you'll want to use a `for` loop to do this.

function myL2(w, b)
    loss = 0.0
    for (x, y) in zip(xs, ys)
        loss += (y - g(x, w, b))^2
    end
    return loss
end

#' Now that you've defined `L2`, we can plot it using either `surface` or
#' `contour` to see how the loss changes as `w` and `b` change!

#' ## plotlyjs() # Use Plotly for 3d surface plots (if possible)

ws = -2:0.05:2
bs = -2:0.05:2

#' ## surface(ws, bs, L2, alpha=0.8, zlims=(0,3))  # alpha gives transparency
contour(ws, bs, L2, levels=0.055:0.05:1.5, xlabel="value of w", ylabel="value of b")


"""
    curve_length(f::Function,a::Real,b::Real;rtol=1e-13) -> Float64

Calculates the length of the curve given by `f`, where `f` is function of one parameter, on interval <`a`, `b`> of said parameter.

# Arguments
- `f::Function`: function of the curve to be measured,
- `a::Real`: left-bound of the interval to be integrated on,
- `b::Real`: right-bound of the interval to be integrated on,
- `rtol::Real`: default `rtol = 1e-13`, relative tolerance,

# Keywords

# Returns
- `Float64`: length of the curve defined by the `f` on the interval <`a`, `b`>.

# Throws
"""
function curve_length(f::Function,a::Real,b::Real;rtol=1e-13)
    df = x -> ForwardDiff.derivative(f,x)
    dr = x -> sqrt(1+df(x)^2)
    l,_ = quadgk(dr,a,b;rtol=rtol)
    return l
end

"""
    curve_fraction(f::Function,a::Real,b::Real,r::Real) -> Float64

Retunrs the position of the fraction of the length from the interval <`a`, `b`>, where the length of the curve is given by one parameter function `f`.

# Arguments
- `f::Function`: function of the curve,
- `a::Real`: left-bound of the interval to be integrated on,
- `b::Real`: right-bound of the interval to be integrated on,
- `r::Real`: fraction of the length of the curve given by `f` from the left,
- `rtol:Real`: default `rtol = 1e-13`, relative tolerance,

# Keywords

# Returns
- `Float64`: position of the fraction of the length from the interval <`a`, `b`>.

# Throws
"""
function curve_fraction(f::Function,a::Real,b::Real,r::Real;rtol=1e-13)
    df = x -> ForwardDiff.derivative(f,x)
    dr = x -> sqrt(1.0 + df(x)^2)

    l,_ = quadgk(dr,a,b;rtol=rtol)
    lf = l*r

    xo = (a+b)/2
    xn = 0.0
    eps_n = 1.0
    while eps_n > xo*rtol
        l,_ = quadgk(dr,a,xo;rtol=rtol)
        xn = xo - (l - lf)/dr(xo)
        eps_n = abs(xo-xn)
        xo = xn
    end

    return xo
end


###############################################################################
#
# Quadratic function solver
#
###############################################################################
"""

    lower_quadratic(a::FT, b::FT, c::FT) where {FT<:AbstractFloat}

Return the lower quadratic solution or NaN, given
- `a` Parameter in `a*x^2 + b*x + c = 0`
- `b` Parameter in `a*x^2 + b*x + c = 0`
- `c` Parameter in `a*x^2 + b*x + c = 0`

---
Example
```julia
low_x = lower_quadratic(1.0, 2.0, 0.0);
low_x = lower_quadratic(-1.0, 2.0, 0.0);
low_x = lower_quadratic(1.0, 2.0, 2.0);
```
"""
function lower_quadratic(a::FT, b::FT, c::FT) where {FT<:AbstractFloat}
    discr = b^2 - 4*a*c;

    if (discr >= 0) && (a > 0)
        return (-b - sqrt(discr)) / (2 * a)
    elseif (discr >= 0) && (a < 0)
        return (-b + sqrt(discr)) / (2 * a)
    else
        return FT(NaN)
    end
end

"""

    upper_quadratic(a::FT, b::FT, c::FT) where {FT<:AbstractFloat}

Return the upper quadratic solution or NaN, given
- `a` Parameter in `a*x^2 + b*x + c = 0`
- `b` Parameter in `a*x^2 + b*x + c = 0`
- `c` Parameter in `a*x^2 + b*x + c = 0`

---
Example
```julia
up_x = upper_quadratic(1.0, 2.0, 0.0);
up_x = upper_quadratic(-1.0, 2.0, 0.0);
up_x = upper_quadratic(1.0, 2.0, 2.0);
```
"""
function upper_quadratic(a::FT, b::FT, c::FT) where {FT<:AbstractFloat}
    discr = b^2 - 4*a*c;

    if (discr >= 0) && (a > 0)
        return (-b + sqrt(discr)) / (2 * a)
    elseif (discr >= 0) && (a < 0)
        return (-b - sqrt(discr)) / (2 * a)
    else
        return FT(NaN)
    end
end


using Optim, HTTP, GLM, LinearAlgebra, Random, Statistics, DataFrames, CSV, FreqTables
using Pkg
Pkg.add("Distributions")
using Pkg
Pkg.add("ForwardDiff")

    #Question1
    using DataFrames
using CSV
using HTTP
url = "https://raw.githubusercontent.com/OU-PhD-Econometrics/fall-2021/master/ProblemSets/PS4-mixture/nlsw88t.csv"
df = CSV.read(HTTP.get(url).body, DataFrame)
X = [df.age df.white df.collgrad]
Z = hcat(df.elnwage1, df.elnwage2, df.elnwage3, df.elnwage4,
df.elnwage5, df.elnwage6, df.elnwage7, df.elnwage8)
y = df.occ_code
#Using Q1 from Pset3
function mlogit_with_Z(theta, X, Z, y)

    alpha = theta[1:end-1]
    gamma = theta[end]
    K = size(X,2)
    J = length(unique(y))
    N = length(y)
    bigY = zeros(N,J)
    for j=1:J
        bigY[:,j] = y.==j
    end
    bigAlpha = [reshape(alpha,K,J-1) zeros(K)]

    T = promote_type(eltype(X),eltype(theta))
    num   = zeros(T,N,J)
    dem   = zeros(T,N)
    for j=1:J
        num[:,j] = exp.(X*bigAlpha[:,j] .+ (Z[:,j] .- Z[:,J])*gamma)
        dem .+= num[:,j]
    end

    P = num./repeat(dem,1,J)

    loglike = -sum( bigY.*log.(P) )

    return loglike
end
startvals = [2*rand(7*size(X,2)).-1; .1]
td = TwiceDifferentiable(theta -> mlogit_with_Z(theta, X, Z, y), startvals; autodiff = :forward)
# run the optimizer
theta_hat_optim_ad = optimize(td, startvals, LBFGS(), Optim.Options(g_tol = 1e-5, iterations=100_000, show_trace=true, show_every=50))
theta_hat_mle_ad = theta_hat_optim_ad.minimizer
# evaluate the Hessian at the estimates
H  = Optim.hessian!(td, theta_hat_mle_ad)
theta_hat_mle_ad_se = sqrt.(diag(inv(H)))
println([theta_hat_mle_ad theta_hat_mle_ad_se]
#Question 2
#Interpretation of gamma#
#Question 3
n.
using Distributions
include("lgwt.jl") # make sure the function gets read in
# define distribution
d = Normal(0,1) # mean=0, standard deviation=1
e.
# get quadrature nodes and weights for 7 grid points
nodes, weights = lgwt(7,-4,4)
# now compute the integral over the density and verify it's 1
sum(weights.*pdf.(d,nodes))
# now compute the expectation and verify it's 0
sum(weights.*nodes.*pdf.(d,nodes))
#computing integrals using the above
d = Normal(0,2)
nodes, weights = lgwt(7,-5,5)
sum(weights.*(nodes.).*pdf.(d,nodes))

d = Normal(0,2)
nodes, weights = lgwt(10,-5,5)
sum(weights.*(nodes.).*pdf.(d,nodes))

#Question4


### A Pluto.jl notebook ###
# v0.16.0

using Markdown
using InteractiveUtils

# ╔═╡ 10349518-03ca-11eb-09b2-69c80c4662ac
begin
	using Pkg
	cd(joinpath(dirname(@__FILE__),".."))
    Pkg.activate(pwd())
	using JuMP, Tulip, GLPK, LinearAlgebra
	using Distributions, Plots, StatsPlots, LaTeXStrings, Measures
end

# ╔═╡ f7f3a256-03c6-11eb-2c1e-83cc62bf55e6
md"""
# Linear programming
We will be using [JuMP](https://jump.dev/JuMP.jl/stable/) as a general framework, combined with [Tulip](https://github.com/ds4dm/Tulip.jl) or [GLPK](https://github.com/jump-dev/GLPK.jl) as a solver.
"""

# ╔═╡ 7910d53e-03ca-11eb-1536-7fc4c236e10a
md"""
## Application - Employee planning
We manage a crew of call center employees and want to optimise our shifts in order to reduce the total payroll cost. Employees have to work for five consecutive days and are then given two days off. The current policy is simple: each day gets the same amount of employees (currently we have 5 persons per shift, which leads to 25 persons on any given day).

We have some historical data that gives us the minimum amount of calls we can expect: Mon: 22, Tue: 17, Wed:13, Thu:14, Fri: 15, Sat: 18, Sun: 24

Employees are payed € 96 per day of work. This lead to the current payroll cost of 25x7x96 = € 16.800. You need to optimize employee planning to reduce the payroll cost.

| Schedule | Days worked | Attibuted Pers | Mon | Tue | Wed | Thu | Fri | Sat | Sun |
|----------|-------------|----------------|-----|-----|-----|-----|-----|-----|-----|
| A | Mon-Fri | 5 | W | W | W | W | W | O | O |
| B | Tue-Sat | 5 | O | W | W | W | W | W | O |
| C | Wed-Sun | 5 | O | O | W | W | W | W | W |
| D | Thu-Mon | 5 | W | O | O | W | W | W | W |
| E | Fri-Tue | 5 | W | W | O | O | W | W | W |
| F | Sat-Wed | 5 | W | W | W | O | O | W | W |
| G | Sun-Thu | 5 | W | W | W | W | O | O | W |
| Totals: | - | 35 | 5 | 5 | 5 | 5 | 5 | 5 | 5 |
| Required: | - | - | 22 | 17 | 13 | 14 | 15 | 18 | 24 |

### Mathematical formulation
We need do formaly define our decision variables, constraints and objective function.
* decision variables: the amount of persons attributed to each schedule ( ``Y = [y_1,y_2,\dots,y_7]^{\intercal} ``)
* objective function: the payroll cost

  Suppose the matrix ``A`` is the matrix indicating the workload for each schedule (in practice ``W=1`` and ``O=0``):
```math
A = \begin{bmatrix}
W & W & W & W & W & O & O \\
O & W & W & W & W & W & O \\
O & O & W & W & W & W & W \\
W & O & O & W & W & W & W \\
W & W & O & O & W & W & W \\
W & W & W & O & O & W & W \\
W & W & W & W & O & O & W 	\\
\end{bmatrix}
```

  Now $$Y*A'$$ gives us a vector indicating the amount of employees working on a given day. Suppose we also use the vector $$c$$ to indicate the salary for a given day (in this case $$c = [96,96,96,\dots,96]$$).

We are now able to write our objective function:
```math
\min Z = c^\intercal A^\intercal Y
```

* constraints (1): each day we need at least enough employees to cover all incoming calls. Suppose we use the vector $$b$$ to indicate the amount of incoming calls for a given day. We are able to write the constraints in a compact way:

```math
\text{subject to } A^\intercal Y  \ge b
```

* constraints (2): we also want to avoid a negative amount of attributed employees on any given day, since this would lead to a negative payroll cost:
```math
\text{and }Y \ge 0
```

* $\forall Y : Y \in \mathbb{N}$
### Implementation
"""

# ╔═╡ 572d14c4-03cb-11eb-2b68-234b3d7e9e8e
begin
	# basic data
	A = ones(Bool,7,7) - diagm(-1=>ones(Bool,6), -2=> ones(Bool,5), 5=>ones(Bool,2), 6=>ones(Bool,1))
	Y = [5,5,5,5,5,5,5]
	B = [22,17,13,14,15,18,24]
	C = [96,96,96,96,96,96,96];
	A
end

# ╔═╡ 4ba4687e-087f-11eb-1cb0-2581818cbd93
# A' * Y .> B
C' * A' * Y

# ╔═╡ 79351ee4-087f-11eb-147d-d389de200857
let
	model = Model(GLPK.Optimizer)
	@variable(model, Y[1:7] >= 0, Int)
	@constraint(model, A' * Y .>= B)
	@objective(model, Min, C' * A' * Y)
	optimize!(model)
	termination_status(model)
	objective_value(model)
	#A' * value.(Y) .> B
	sum(value.(Y))
end

# ╔═╡ 3bafd152-6724-4847-a79d-f2340e2ab5e4

# ╔═╡ 4bc4c2aa-04b4-11eb-2b6e-452d4ecc258a
md"""
### Adding uncertainty
Up to now, we have had constant numbers the minimum number of employees needed per day. In reality these quantities are uncertain. The actual number of calls will fluctuate each day. For simplicity's sake will we use a [lognormal distribution](https://en.wikipedia.org/wiki/Log-normal_distribution#Occurrence_and_applications) for the amount of calls (using their initial value as mean and a standard deviation of two). Working this way, we avoid having negative calls.
"""

# ╔═╡ 6e48e306-04b4-11eb-2561-0151a5e0a908
B

# ╔═╡ 7a54f9b4-04b4-11eb-3a7c-8d90eb026392
begin
	# generating the distributions
	B_u = Distributions.LogNormal.(B,2) # array with distributions

	# quick sample to illustrate amount of calls being randomized
	log.(rand.(B_u))
end

# ╔═╡ 9d3ceafe-0880-11eb-36f0-0f8530d6b285
begin
	cost = Float64[]
	for _ in 1:10000
		let cout=cost
			model = Model(GLPK.Optimizer)
			@variable(model, Y[1:7] >= 0, Int)
			@constraint(model, A' * Y .>= log.(rand.(B_u)))
			@objective(model, Min, C' * A' * Y)
			optimize!(model)
			push!(cout, objective_value(model))
		end
	end
	cost
end

# ╔═╡ 14a32a20-0881-11eb-1a20-715b99417266
StatsPlots.histogram(cost, yscale=:log10)

# ╔═╡ 259bbbc6-04b5-11eb-1ad4-c567e45ba4b6
md"""
### Small variant: adding a commission
Suppose each worker receives extra pay for the amount of calls that have been treated. We can easily include this in our model
"""

# ╔═╡ 83130656-0881-11eb-2c87-a3ae5f81f04b
begin
	cost_c = Float64[]
	commission = 20
	for _ in 1:1000
		let cout=cost_c
			model = Model(GLPK.Optimizer)
			appels = log.(rand.(B_u))
			@variable(model, Y[1:7] >= 0, Int)
			@constraint(model, A' * Y .>= appels)
			@objective(model, Min, C' * A' * Y + sum(appels) * commission)
			optimize!(model)
			push!(cout, objective_value(model))
		end
	end
	StatsPlots.histogram(cost_c)
end

# ╔═╡ e1e41ea6-04b5-11eb-174e-1d43f601a07c
md"""
#### Playing it safe
The above has given us some information on what the distributions of the payroll cost may be, however in reality, you would want to make sure that the clients calling to center are taken care off. To realise this, one might say that for any given day, you want to make sure that 90% of all calls can be treated by the specific capacity.
"""

# ╔═╡ 62b516be-0882-11eb-094b-d183f8d00ab8
log.(quantile.(B_u, 0.90))

# ╔═╡ 6274be66-0882-11eb-11be-05163cda6633
 let
	model = Model(GLPK.Optimizer)
	@variable(model, Y[1:7] >= 0, Int)
	@constraint(model, A' * Y .>= log.(quantile.(B_u, 0.99)))
	@objective(model, Min, C' * A' * Y)
	optimize!(model)
	termination_status(model)
	objective_value(model)
end

# ╔═╡ 1f177098-04b6-11eb-2508-6bd8d7e1e996
md"""
### Additional questions
* The example we have treated so far has very traditional working patterns for the employees. How woud you deal with modern working patterns (e.g. 4/5 or parttime working)?
* We took a look at the stochastic nature of the amount of calls, however, the personnel might not show up for various reasons. How would you describe the possible influence? Hint: make a discrete event model of this setting, using the optimal design and controlling for employees showing up or not.
"""

# ╔═╡ Cell order:
# ╟─f7f3a256-03c6-11eb-2c1e-83cc62bf55e6
# ╠═10349518-03ca-11eb-09b2-69c80c4662ac
# ╟─7910d53e-03ca-11eb-1536-7fc4c236e10a
# ╠═572d14c4-03cb-11eb-2b68-234b3d7e9e8e
# ╠═4ba4687e-087f-11eb-1cb0-2581818cbd93
# ╠═79351ee4-087f-11eb-147d-d389de200857
# ╠═3bafd152-6724-4847-a79d-f2340e2ab5e4
# ╟─4bc4c2aa-04b4-11eb-2b6e-452d4ecc258a
# ╠═6e48e306-04b4-11eb-2561-0151a5e0a908
# ╠═7a54f9b4-04b4-11eb-3a7c-8d90eb026392
# ╠═9d3ceafe-0880-11eb-36f0-0f8530d6b285
# ╠═14a32a20-0881-11eb-1a20-715b99417266
# ╟─259bbbc6-04b5-11eb-1ad4-c567e45ba4b6
# ╠═83130656-0881-11eb-2c87-a3ae5f81f04b
# ╟─e1e41ea6-04b5-11eb-174e-1d43f601a07c
# ╠═62b516be-0882-11eb-094b-d183f8d00ab8
# ╠═6274be66-0882-11eb-11be-05163cda6633
# ╟─1f177098-04b6-11eb-2508-6bd8d7e1e996

Exercises/PS05 - Linear programming I.jl


module Bisect

export idx_of_closest
#export bin_search, idx_of_closest

function midpoint(a, b)
    if a > b
        error("... b must be greater or equal than a; a == $a, b == $b")
    elseif a == b
        a
    else
        div(a + b , 2)
    end
end

#................ 1) Search bin
function search_bin(edges::Vector{Float64}, val::Float64, imin::Int, imax::Int)
    # find the BIN - characterized by the pair of edges
    # If
    imid = midpoint(imin, imax)
    if imin == imax
        if val >= edges[imid]
            imid + 1
        else
            imid
        end
    else
        if val >= edges[imid]
            search_bin(edges, val, imid + 1, imax)
        else
            search_bin(edges, val, imin, max(imid - 1, imin))
        end
    end
end

function search_bin(edges::Vector{Float64}, val::Float64)
    search_bin(edges, val, 1, length(edges))
end

function test_search_bin()
    println( search_bin([0.1, 0.2, 0.3], 0.05)) # 1
    println( search_bin([0.1, 0.2, 0.3], 0.10)) # 2
    println( search_bin([0.1, 0.2, 0.3], 0.15)) # 2
    println( search_bin([0.1, 0.2, 0.3], 0.19)) # 2
    println( search_bin([0.1, 0.2, 0.3], 0.29)) # 3
    println( search_bin([0.1, 0.2, 0.3], 0.31)) # 4
end
# test_search_bin()
#................ End 1) Search bin
#............... 2) Index of the closest element
function idx_of_closest(edges::Vector{Float64}, val::Float64, imin::Int, imax::Int)
    # Find index of the closest edge to the value
    bin  = search_bin(edges, val)
    n = length(edges)
    if bin == 1
        edges[bin]
    elseif bin == n + 1
        edges[n]
    else
        left_dist  = abs(val - edges[bin-1])
        right_dist = abs(val - edges[bin]  )
        left_dist >= right_dist ? bin : bin - 1
    end
end

function idx_of_closest(edges::Vector{Float64}, val::Float64)
    idx_of_closest(edges, val, 1, length(edges))
end

function test_bin_search_closest_idx()
    println( idx_of_closest([0.1, 0.2, 0.5], 0.15))
    println( idx_of_closest([0.1, 0.2, 0.5], 0.16))
end
#test_bin_search_closest_idx()
#............... End 2) Index of the closest element
end


# ## Line plot 2d

#md # ![FILE_NAME.png](images/FILE_NAME.png)

x = 0:0.1:2pi
y1 = cos.(x)
y2 = sin.(x)

plot(x, y1, c="blue", linewidth=3)
plot!(x, y2, c="red", line=:dash)
title!("Trigonometric functions")
xlabel!("angle")
ylabel!("sin(x) and cos(x)")

## axis limit
plot!(xlims=(0,2pi), ylims=(-2, 2))

#-
## same plot
x = 0:0.1:2pi
y1 = cos.(x)
y2 = sin.(x)

plot(x, y1,
    c="blue",
    linewidth=3,
    title="Trigonometric functions",
    xlabel="angle",
    ylabel="sin(x) and cos(x)")
plot!(x, y2, c="red", line=:dash)

plot!(xlims=(0,2pi), ylims=(-2, 2))


"""
hatfun(x,t,k)

Evaluate a piecewise linear "hat" function at `x`, where `t` is a
vector of n+1 interpolation nodes and `k` is an integer in 0:n
giving the index of the node where the hat function equals one.
"""

function hatfun(x,t,k)
    n = length(t)-1
    # Return correct node given mathematical index k, including
    # fictitious choices.
    function node(k)
        if k < 0
            2t[1]-t[2]
        elseif k > n
            2t[n+1]-t[n]
        else
            t[k+1]
        end
    end

    H1 = (x-node(k-1))/(node(k)-node(k-1))   # upward slope
    H2 = (node(k+1)-x)/(node(k+1)-node(k))   # downward slope

    H = min(H1,H2)
    return max(0,H)
end

"""
plinterp(t,y)

Create a piecewise linear interpolating function for data values in
`y` given at nodes in `t`.
"""
function plinterp(t,y)
n = length(t)-1
return x -> sum( y[k+1]*hatfun(x,t,k) for k in 0:n )
end

"""
spinterp(t,y)

Create a cubic not-a-knot spline interpolating function for data
values in `y` given at nodes in `t`.
"""
function spinterp(t,y)

    n = length(t)-1
    h = diff(t)         # differences of all adjacent pairs

    # Preliminary definitions.
    Z = zeros(n,n);
    In = I(n);  E = In[1:n-1,:];
    J = diagm(0=>ones(n),1=>-ones(n-1))
    H = diagm(0=>h)

    # Left endpoint interpolation:
    AL = [ In Z Z Z ]
    vL = y[1:n]

    # Right endpoint interpolation:
    AR = [ In H H^2 H^3 ];
    vR = y[2:n+1]

    # Continuity of first derivative:
    A1 = E*[ Z J 2*H 3*H^2 ]
    v1 = zeros(n-1)

    # Continuity of second derivative:
    A2 = E*[ Z Z J 3*H ]
    v2 = zeros(n-1)

    # Not-a-knot conditions:
    nakL = [ zeros(1,3*n) [1 -1 zeros(1,n-2)] ]
    nakR = [ zeros(1,3*n) [zeros(1,n-2) 1 -1] ]

    # Assemble and solve the full system.
    A = [ AL; AR; A1; A2; nakL; nakR ]
    v = [ vL; vR; v1; v2; 0; 0 ]
    z = A\v

    # Break the coefficients into separate vectors.
    rows = 1:n
    a = z[rows]
    b = z[n.+rows];  c = z[2*n.+rows];  d = z[3*n.+rows]
    S = [ Polynomial([a[k],b[k],c[k],d[k]]) for k = 1:n ]
    # This function evaluates the spline when called with a value
    # for x.
    function evaluate(x)
        k = findfirst(@. x<t)   # one greater than interval x belongs to
        k==1 && return NaN
        if isnothing(k)
            return x==t[end] ? y[end] : NaN
        end
        return S[k-1](x-t[k-1])
    end
    return evaluate
end

"""
fdweights(t,m)

Return weights for the `m`th derivative of a function at zero using
values at the nodes in vector `t`.
"""
function fdweights(t,m)
    # This is a compact implementation, not an efficient one.

    function weight(t,m,r,k)
        # Recursion for one weight. Input: t   nodes (vector) m
        # order of derivative sought r   number of nodes to use from
        # t (<= length(t)) k   index of node whose weight is found

        if (m<0) || (m>r)        # undefined coeffs must be zero
            c = 0
        elseif (m==0) && (r==0)  # base case of one-point interpolation
            c = 1
        else                     # generic recursion
            if k<r
                c = (t[r+1]*weight(t,m,r-1,k) -
                    m*weight(t,m-1,r-1,k))/(t[r+1]-t[k+1])
            else
                numer = r > 1 ? prod(t[r]-x for x in t[1:r-1]) : 1.0
                denom = r > 0 ? prod(t[r+1]-x for x in t[1:r]) : 1.0
                β = numer/denom
                c = β*(m*weight(t,m-1,r-1,r-1) - t[r]*weight(t,m,r-1,r-1))
            end
        end
        return c
    end

    r = length(t)-1
    w = zeros(size(t))
    return [ weight(t,m,r,k) for k=0:r ]
end

"""
trapezoid(f,a,b,n)

Apply the trapezoid integration formula for integrand `f` over
interval [`a`,`b`], broken up into `n` equal pieces. Returns
estimate, vector of nodes, and vector of integrand values at the
nodes.
"""
function trapezoid(f,a,b,n)
    h = (b-a)/n
    t = LinRange(a,b,n+1)
    y = f.(t)
    T = h * ( sum(y[2:n]) + 0.5*(y[1] + y[n+1]) )

    return T,t,y
end

"""
intadapt(f,a,b,tol)

Do adaptive integration to estimate the integral of `f` over
[`a`,`b`] to desired error tolerance `tol`. Returns estimate and a
vector of evaluation nodes used.
"""
function intadapt(f,a,b,tol)
    # Use error estimation and recursive bisection.
    function do_integral(a,fa,b,fb,m,fm,tol)
        # These are the two new nodes and their f-values.
        xl = (a+m)/2;  fl = f(xl);
        xr = (m+b)/2;  fr = f(xr);
        t = [a,xl,m,xr,b]          # all 5 nodes at this level

        # Compute the trapezoid values iteratively.
        h = (b-a)
        T = [0.,0.,0.]
        T[1] = h*(fa+fb)/2
        T[2] = T[1]/2 + (h/2)*fm
        T[3] = T[2]/2 + (h/4)*(fl+fr)

        S = (4*T[2:3]-T[1:2]) / 3      # Simpson values
        E = (S[2]-S[1]) / 15           # error estimate

        if abs(E) < tol*(1+abs(S[2]))  # acceptable error?
            Q = S[2]                   # yes--done
        else
            # Error is too large--bisect and recurse.
            QL,tL = do_integral(a,fa,m,fm,xl,fl,tol)
            QR,tR = do_integral(m,fm,b,fb,xr,fr,tol)
            Q = QL + QR
            t = [tL;tR[2:end]]   # merge the nodes w/o duplicate
        end
        return Q,t
    end

    m = (b+a)/2
    Q,t = do_integral(a,f(a),b,f(b),m,f(m),tol)
    return Q,t
end


"""
    forwardsub(L,b)

Solve the lower-triangular linear system with matrix `L` and right-hand side
vector `b`.
"""
function forwardsub(L,b)

    n = size(L,1)
    x = zeros(n)
    for i = 1:n
        s = i > 1 ? sum( L[i,j]*x[j] for j=1:i-1 ) : 0
        x[i] = ( b[i] - s ) / L[i,i]
    end

    return x
end

"""
    backsub(U,b)

Solve the upper-triangular linear system with matrix `U` and right-hand side
vector `b`.
"""
function backsub(U,b)

    n = size(U,1)
    x = zeros(n)
    for i = n:-1:1
        s = i < n ? sum( U[i,j]*x[j] for j=i+1:n ) : 0
        x[i] = ( b[i] - s ) / U[i,i]
    end

    return x
end

"""
    lufact(A)

Compute the LU factorization of square matrix `A`, returning the factors.
"""
function lufact(A)

    n = size(A,1)
    L = Matrix(Diagonal(ones(n)))    # puts ones on diagonal
    U = copy(A)

    # Gaussian elimination
    for j = 1:n-1
        for i = j+1:n
            L[i,j] = U[i,j] / U[j,j]   # row multiplier
            U[i,j:n] = U[i,j:n] - L[i,j]*U[j,j:n]
        end
    end

    return L,triu(U)
end


using LinearAlgebra

function classical_gram_schmidt(A)
    m, n = size(A)
    R = zeros(n, n)
    Q = zeros(m, n)
    for j in 1:n
        v_j = A[:, j]
        for i in 1:(j-1)
            R[i, j] = dot(Q[:, i], A[:, j])
            v_j = v_j - R[i, j] * Q[:, i]
        end
        R[j, j] = norm(v_j)
        Q[:, j] = v_j / R[j, j]
    end
    return Q, R
end

function modified_gram_schmidt(A)
    m, n = size(A)
    R = zeros(Float64, n, n)
    Q = zeros(Float64, m, n)

    V = zeros(Float64, n, n)
    for i in 1:n
        V[:, i] = A[:, i]
    end

    for i in 1:n
        R[i, i] = norm(V[:, i])
        Q[:, i] = V[:, i] / R[i, i]
        for j in (i+1):n
            R[i, j] = dot(Q[:, i], V[:, j])
            V[:, j] = V[:, j] - R[i, j] * Q[:, i]
        end
    end
    return Q, R
end

function check_stability(Q)
    m, n = size(Q)
    total_dot_error = 0
    for i in 1:m
        for j in 1:n
            if i != j
                total_dot_error += abs(dot(Q[:, i], Q[:, j]))
            end
        end
    end
    return total_dot_error
end


"""
Estimate if the number is prime
"""
function check_composite(a::BigInt, d::BigInt, n::BigInt, s::BigInt)
    x = exp!(a, d, n)

    if (x == 1 || x == (n - 1))
        return (false)
    end

    for _ in 1:s
        x = (x * x) % n
        if (x == (n - 1))
            return (false)
        end
    end
    true
end

check_composite(a::T, d::T, n::T, s::T) where {T <: Integer} =
check_composite(big(a), big(d), big(n), big(s))

"""Miller Rabin primality test
Return False if n is composite, True(probably prime) otherwise.

n: Number to be tested for primality
k: number of iterations to run the test
"""
function miller_rabin(n::BigInt, k::BigInt = big(10))
    if (n == 2)
        return (true)
    end
    if (n == 1 || n % 2 == 0)
        return (false)
    end

    s::BigInt = 0
    d = n - 1

    while d % 2 == 0
        s += 1
        d /= 2
    end

    @assert floor(BigInt, 2 ^ s * d) == n - 1

    for _ in 1:k
        a = rand(2:n - 1)
        d = floor(BigInt, d)
        if (check_composite(a, d, n, s))
            return (false)
        end
    end
    true
end

miller_rabin(n::T, k::T = 10) where {T <: Integer} =
miller_rabin(big(n), big(k))

"""
Solovay Strassen Primality Test
Returns False is n is composite, True(probably prime) otherwise

n: Number to be tested for primality
k: number of iterations to run the test
"""
function solovay_strassen(n::BigInt, k::BigInt = big(10))
    if (n == 2)
        return (true)
    end
    if (n == 1 || n & 1 == 0)
        return (false)
    end

    for _ in 1:k
        a = rand(2:n-1)
        x = (jacobi!(a, n) + n) % n
        if (x == 0 || exp!(a, floor(BigInt, (n - 1) / 2), n) != x)
            return (false)
        end
    end
    true
end

solovay_strassen(n::T, k::T = 10) where {T <: Integer} =
solovay_strassen(big(n), big(k))

"""
Fermat's Primality test
Returns True(probably prime) if n is prime, Flase if n is composite

n: Number to be tested for primality
k: number of iterations of the Fermat's Primality Test
"""
function fermat_test(n::BigInt, k::BigInt = big(10))
    if (n == 2)
        return (true)
    end
    if (n & 1 == 0)
        return (false)
    end

    for _ in 1:k
        a = rand(2:n-1)
        if (exp!(a, n - 1, n) != 1)
            return (false)
        end
    end
    true
end

fermat_test(n::T, k::T = 10) where {T <: Integer} =
fermat_test(big(n), big(k))


export SoftMax

"""
`SoftMax([domainType=Float64::Type,] dim_in::Tuple)`

Creates the softmax non-linear operator with input dimensions `dim_in`.
```math
\\sigma(\\mathbf{x}) = \\frac{e^{\\mathbf{x} }}{ \\sum e^{\\mathbf{x} } }
```

"""
struct SoftMax{T,N} <: NonLinearOperator
	dim::NTuple{N,Int}
	buf::Array{T,N}
end

function SoftMax(DomainType::Type, DomainDim::NTuple{N,Int}) where {N}
	SoftMax{DomainType,N}(DomainDim,zeros(DomainType,DomainDim))
end

SoftMax(DomainDim::NTuple{N,Int}) where {N} = SoftMax(Float64,DomainDim)

function mul!(y::AbstractArray{T,N}, L::SoftMax{T,N}, x::AbstractArray{T,N}) where {T,N}
	y .= exp.(x.-maximum(x))
	y ./= sum(y)
end

function mul!(y::AbstractArray,
              J::AdjointOperator{Jacobian{A,TT}},
              b::AbstractArray) where {T, N, A<: SoftMax{T,N}, TT <: AbstractArray{T,N} }
    L = J.A
	fill!(y,zero(T))
	L.A.buf .= exp.(L.x.-maximum(L.x))
	L.A.buf ./= sum(L.A.buf)
	for i in eachindex(y)
		y[i] = -L.A.buf[i]*dot(L.A.buf,b)
		y[i] += L.A.buf[i]*b[i]
	end
	return y
end

fun_name(L::SoftMax) = "σ"

size(L::SoftMax) = (L.dim, L.dim)

domainType(L::SoftMax{T,N}) where {T,N} = T
codomainType(L::SoftMax{T,N}) where {T,N} = T


@doc doc"""
Approximation de la solution du problème ``\min_{x \in \mathbb{R}^{n}} f(x)`` en utilisant l'algorithme de Newton

# Syntaxe
```julia
xk,f_min,flag,nb_iters = Algorithme_de_Newton(f,gradf,hessf,x0,option)
```

# Entrées :
   * **f**       : (Function) la fonction à minimiser
   * **gradf**   : (Function) le gradient de la fonction f
   * **hessf**   : (Function) la Hessienne de la fonction f
   * **x0**      : (Array{Float,1}) première approximation de la solution cherchée
   * **options** : (Array{Float,1})
       * **max_iter**      : le nombre maximal d'iterations
       * **Tol_abs**       : la tolérence absolue
       * **Tol_rel**       : la tolérence relative

# Sorties:
   * **xmin**    : (Array{Float,1}) une approximation de la solution du problème  : ``\min_{x \in \mathbb{R}^{n}} f(x)``
   * **f_min**   : (Float) ``f(x_{min})``
   * **flag**     : (Integer) indique le critère sur lequel le programme à arrêter
      * **0**    : Convergence
      * **1**    : stagnation du xk
      * **2**    : stagnation du f
      * **3**    : nombre maximal d'itération dépassé
   * **nb_iters** : (Integer) le nombre d'itérations faites par le programme

# Exemple d'appel
```@example
using Optinum
f(x)=100*(x[2]-x[1]^2)^2+(1-x[1])^2
gradf(x)=[-400*x[1]*(x[2]-x[1]^2)-2*(1-x[1]) ; 200*(x[2]-x[1]^2)]
hessf(x)=[-400*(x[2]-3*x[1]^2)+2  -400*x[1];-400*x[1]  200]
x0 = [1; 0]
options = []
xmin,f_min,flag,nb_iters = Algorithme_De_Newton(f,gradf,hessf,x0,options)
```
"""
function Algorithme_De_Newton(f::Function,gradf::Function,hessf::Function,x0,options)

    "# Si options == [] on prends les paramètres par défaut"
    if options == []
        max_iter = 100
        Tol_abs = sqrt(eps())
        Tol_rel = 1e-15
    else
        max_iter = options[1]
        Tol_abs = options[2]
        Tol_rel = options[3]
    end

    n = length(x0)
    xmin = x0
    f_min = 0
    flag = 0
    nb_iters = 0
    xk = x0
    xk_1 = x0

    while (norm(gradf(xk)) > max(Tol_rel * norm(gradf(x0)), Tol_abs))
        nb_iters += 1
        xk_1 = xk
        dk = -hessf(xk) \ gradf(xk)
        xk = xk + dk

        if norm(gradf(xk_1)) <= max(Tol_abs, Tol_rel * norm(gradf(x0)))
            flag = 0
            break

        elseif norm(dk) <= max(Tol_abs, Tol_rel * norm(xk))
            flag = 1
            break

        elseif abs(f(xk_1) - f(xk)) <= max(Tol_abs, Tol_rel * abs(f(xk)))
            flag = 2
            break
        elseif nb_iters == max_iter
            flag = 3
            break
        end
    end

    xmin = xk
    f_min = f(xmin)

    return xmin,f_min,flag,nb_iters
end


function Example_4_3(; T = 200, seed = 1)

    # Set random number generator

    rg = MersenneTwister(seed)

    # state x is four dimensional, comprises car position and velocities

    Δt  = 1 / 10
    σ₁  = σ₂  = 1 / 2
    qᶜ₁ = qᶜ₂ = 1

    # Matrices in the dynamic model are:

    A = [1  0  Δt  0;
         0  1   0  Δt;
         0  0   1  0;
         0  0   0  1]

    Q = [(qᶜ₁*Δt^3)/3         0        (qᶜ₁*Δt^2)/2       0        ;
              0         (qᶜ₂*Δt^3)/3         0        (qᶜ₁*Δt^2)/2 ;
         (qᶜ₁*Δt^2)/2         0            qᶜ₁*Δt         0        ;
              0         (qᶜ₂*Δt^2)/2         0          qᶜ₂*Δt     ]

    # Measurement matrices

    H = [1 0 0 0;
         0 1 0 0]

    R = [σ₁^2  0;
         0    σ₂^2]

    # initialise

    m₀, P₀ = zeros(4), 0.1*Matrix(I, 4, 4)

    x₀ = rand(MvNormal(m₀, P₀))

    # store measurements and states

    yclean = Array{Vector{Float64},1}(undef, T)

    y = Array{Vector{Float64},1}(undef, T)

    x = Array{Vector{Float64},1}(undef, T)

    # first state

    x[1] = A * x₀

    y[1] = rand(rg, MvNormal(H * x[1], R))

    for k in 2:T

        x[k] = rand(rg, MvNormal(A * x[k-1], Q)) # (4.32)

        yclean[k] = H * x[k]

        y[k] = rand(rg, MvNormal(H * x[k], R)) # (4.33)

    end

    return yclean, y, x, A, H, Q, R, m₀, P₀

end

function runExample_4_3(seed = 1)

    # Simulate data

    yclean, y, x, A, H, Q, R, m₀, P₀ = Example_4_3(seed=seed)

    # Inference

    μfilter, Σfilter = filteringrecursion(y; A = A, H = H, Q = Q, R = R, m₀ = m₀, P₀ = P₀)

    μsmooth, Σsmooth = smoothingrecursion(y; A = A, H = H, Q = Q, R = R, m = μfilter, P = Σfilter)

    # collect positions

    truepositions = reduce(hcat, [xₖ[1:2] for xₖ in x])

    filtpositions = reduce(hcat, [xₖ[1:2] for xₖ in μfilter])

    smoopositions = reduce(hcat, [xₖ[1:2] for xₖ in μsmooth])

    # plot positions

    figure(1)

    subplot(211); cla()

    plot(truepositions[1,:], truepositions[2,:], "o", label = "true positions")

    plot(filtpositions[1,:], filtpositions[2,:], ".", label = "filter positions", alpha=0.5)

    legend()

    subplot(212); cla()

    plot(truepositions[1,:], truepositions[2,:], "o", label = "true positions")

    plot(smoopositions[1,:], smoopositions[2,:], ".", label = "smooth positions", alpha=0.5)

    legend()

    nothing

end


function Primes(N::Int64)

	isprime = ones(Bool,N)

	for i = 2:int64(sqrt(N))
		if isprime[i]
			for j in (i*i):i:N
				isprime[j] = false
			end
		end
	end
	return filter(x -> isprime[x], 2:N)
end

function TrialDivision(N::Int64)

	primeFactors = [1]

	testPrimes = Primes(N);
	Nc = N;
	idx = 1;
	while (prod(primeFactors)!=N) && (idx < length(testPrimes))
		if mod(Nc, testPrimes[idx])==0
			println(testPrimes[idx])
			primeFactors=[primeFactors,testPrimes[idx]]
			Nc = Nc/testPrimes[idx]
		end
		idx = idx+1
	end

	return primeFactors[2:end]
end

function LargestPrimeFactor(N::Int64)

	first20 = Primes(1000)
	largest = 1
	for p in first20
		if mod(N, p) == 0
			largest = p
			N = fld(N,p)
		end
	end
	if N == 1
		return largest
	end
	trials = flipud(Primes(N+1))
	for p in trials
		if (mod(N,p) == 0)&&(p > largest)
			println(p)
			largest = p
			break
		end
	end
	if largest == 1
		return N
	else
		return largest
	end
end

function Euler3(N::Int64)

	big = LargestPrimeFactor(N)
	println("The largest prime factor of ",N," is ",big,".")

end

@time Euler3(600851475143)
#Euler3(100)


"""
    bisection(f,a,b)

adalah fungsi yang mencari nilai akar persamaan dari fungsi `f(x)` pada interval
di antara `a` dan `b`.

# Examples
```jldoctest
julia> f(x) = x*sin(x)-1;

julia> c, flag, M = bisection(f,0,2);

julia> c
1.1141571998596191

julia> flag
0

julia> M
22×5 Array{Float64,2}:
  0.0  0.0      1.0      2.0      -0.158529
  1.0  1.0      1.5      2.0       0.496242
  2.0  1.0      1.25     1.5       0.186231
  3.0  1.0      1.125    1.25      0.015051
  ⋮
 18.0  1.11415  1.11415  1.11416  -3.22926e-6
 19.0  1.11415  1.11416  1.11416  -5.80313e-7
 20.0  1.11416  1.11416  1.11416   7.44159e-7
 21.0  1.11416  1.11416  1.11416   8.19227e-8
```
return solusi `c` dengan `flag` bernilai 0 jika metode bisection berhasil menemukan
solusi dan gagal jika tidak 0. Serta, matriks `M` yang berisi catatan proses tiap
iterasi `[k, a, c, b, f(c)]`

"""
function bisection(f,a,b)
    delta = 10^-7; maxi = 100; flag = 1;
    M = Array{Float64}(undef, 0, 5);
    fa = f(a); fb = f(b);
    if fa*fb>0
        c = "error: f(a) dan f(b) harus berbeda tanda";
        flag = 2;
        return;
    end
    k = 1
    while k<=maxi
        c  = (b+a)/2;
        fc = f(c);
        M = [M; [k-1 a c b fc] ];
        if fc == 0
            a = c;
            b = c;
        elseif fa*fc>0
            a = c;
            fa = fc;
        else
            b = c;
            fb = fc;
        end
        if b-a < delta || abs(fc) < delta
            flag = 0;
            break;
        end
        k+=1
    end
    return c,flag, M
end


using LinearAlgebra
using SparseArrays
using Plots
using ForwardDiff

"This code applies the method of manufactured solutions to the 2D Laplace's equation
with nonzero Dirichlet BCs using a finite difference method."

m = 100 # number of points

uexact(x,y) = log(2+sin(pi*x)*sin(pi*y))
dudx_exact(x,y) = ForwardDiff.derivative(x->uexact(x,y),x)
dudy_exact(x,y) = ForwardDiff.derivative(y->uexact(x,y),y)
du2dx2_exact(x,y) = ForwardDiff.derivative(x->dudx_exact(x,y),x)
du2dy2_exact(x,y) = ForwardDiff.derivative(y->dudy_exact(x,y),y)
f(x,y) = -(du2dx2_exact(x,y) + du2dy2_exact(x,y))

u_left(y)   = uexact(-1,y)
u_right(y)  = uexact(1,y)
u_bottom(x) = uexact(x,-1)
u_top(x)    = uexact(x,1)
uBC = (u_left,u_right,u_bottom,u_top)

function solve(m,f,uBC)
    u_left,u_right,u_bottom,u_top = uBC
    x = LinRange(-1,1,m+2)
    y = LinRange(-1,1,m+2)
    xint = x[2:end-1]
    yint = y[2:end-1]
    h = x[2]-x[1]

    id(i,j,m) = i + (j-1)*m # return global index from i,j

    function FD_matrix_2D(h,m)
        A = spzeros(m*m,m*m)
        for i = 1:m, j = 1:m # loop thru x,y indices
            A[id(i,j,m),id(i,j,m)] = 4/h^2
            if i > 1 # avoids leftmost line of nodes
                A[id(i,j,m),id(i-1,j,m)] = -1/h^2 # x-derivative
            end
            if i < m # avoids rightmost line of nodes
                A[id(i,j,m),id(i+1,j,m)] = -1/h^2
            end
            if j > 1
                A[id(i,j,m),id(i,j-1,m)] = -1/h^2 # y-derivative
            end
            if j < m
                A[id(i,j,m),id(i,j+1,m)] = -1/h^2
            end
        end
        return A
    end

    A = FD_matrix_2D(h,m)

    b = vec(f.(xint,yint'))

    # impose non-zero Dirichlet BCs
    for i = 1:m, j = 1:m
        if i==1
            b[id(i,j,m)] += u_left(yint[j])/h^2
        end
        if i==m
            b[id(i,j,m)] += u_right(yint[j])/h^2
        end
        if j==1
            b[id(i,j,m)] += u_bottom(xint[i])/h^2
        end
        if j==m
            b[id(i,j,m)] += u_top(xint[i])/h^2
        end
    end

    # solve Au = f(xi,yj)
    u = A\b
    return u,h,xint,yint
end

errvec = []
hvec = []
for m in [4 8 16 32 64 128]
    u,h,xint,yint = solve(m,f,uBC)
    err = maximum(abs.(u .- vec(uexact.(xint,yint'))))
    append!(errvec,err)
    append!(hvec,h)
end
plot(hvec,errvec,mark=:dot,xaxis=:log,yaxis=:log)
plot!(hvec,hvec.^2,ls=:dash,xaxis=:log,yaxis=:log)
# contourf(xint,yint,))


#=

The four adjacent digits in the 1000-digit number that have the greatest product are 9 × 9 × 8 × 9 = 5832.

73167176531330624919225119674426574742355349194934
96983520312774506326239578318016984801869478851843
85861560789112949495459501737958331952853208805511
12540698747158523863050715693290963295227443043557
66896648950445244523161731856403098711121722383113
62229893423380308135336276614282806444486645238749
30358907296290491560440772390713810515859307960866
70172427121883998797908792274921901699720888093776
65727333001053367881220235421809751254540594752243
52584907711670556013604839586446706324415722155397
53697817977846174064955149290862569321978468622482
83972241375657056057490261407972968652414535100474
82166370484403199890008895243450658541227588666881
16427171479924442928230863465674813919123162824586
17866458359124566529476545682848912883142607690042
24219022671055626321111109370544217506941658960408
07198403850962455444362981230987879927244284909188
84580156166097919133875499200524063689912560717606
05886116467109405077541002256983155200055935729725
71636269561882670428252483600823257530420752963450

Find the thirteen adjacent digits in the 1000-digit number that have the greatest product. What is the value of this product?

=#

str ="""73167176531330624919225119674426574742355349194934
        96983520312774506326239578318016984801869478851843
        85861560789112949495459501737958331952853208805511
        12540698747158523863050715693290963295227443043557
        66896648950445244523161731856403098711121722383113
        62229893423380308135336276614282806444486645238749
        30358907296290491560440772390713810515859307960866
        70172427121883998797908792274921901699720888093776
        65727333001053367881220235421809751254540594752243
        52584907711670556013604839586446706324415722155397
        53697817977846174064955149290862569321978468622482
        83972241375657056057490261407972968652414535100474
        82166370484403199890008895243450658541227588666881
        16427171479924442928230863465674813919123162824586
        17866458359124566529476545682848912883142607690042
        24219022671055626321111109370544217506941658960408
        07198403850962455444362981230987879927244284909188
        84580156166097919133875499200524063689912560717606
        05886116467109405077541002256983155200055935729725
        71636269561882670428252483600823257530420752963450"""
long_string = replace(str, "\n" => "")
len_adjacent = 13

function largest_adjacent_product(string, len_adjacent)
    num_digits = length(string)
    largest_product = 0
    largest_piece = 0
    for offset in 0:num_digits-len_adjacent
        number_piece = string[1+offset:len_adjacent+offset]
        #println(number_piece)
        product = prod(digits(parse(Int, number_piece)))
        if product > largest_product
            largest_product = product
            largest_piece = number_piece
        end

    end
    return largest_product, largest_piece
end

println()
print(largest_adjacent_product(long_string, len_adjacent))


function sinh(x::DoubleFloat{T}) where {T<:IEEEFloat}
    x < 0 && return -sinh(-x)
    iszero(x) && return zero(x)
    !isfinite(x) && return nan(typeof(x))

    y = exp(x) - exp(-x)
    z = DoubleFloat{T}(y.hi*0.5, y.lo*0.5)
    return z
end

function cosh(x::DoubleFloat{T}) where {T<:IEEEFloat}
    x < 0 && return cosh(-x)
    iszero(x) && return one(x)
    !isfinite(x) && return nan(typeof(x))

    y = exp(x) + exp(-x)
    z = DoubleFloat{T}(y.hi*0.5, y.lo*0.5)
    return z
end

function tanh(x::DoubleFloat{T}) where {T<:IEEEFloat}
    return sinh(x) / cosh(x)
end
#=
function tanh(x::DoubleFloat{T}) where {T<:IEEEFloat}
    x < 0 && return -tanh(-x)
    iszero(x) && return zero(x)
    !isfinite(x) && return nan(typeof(x))
    epos = exp(x)
    eneg = exp(-x)
    z = (epos - eneg) / (epos + eneg)
    return z
end
=#

csch(x::DoubleFloat{T}) where {T<:IEEEFloat} =
    inv(sinh(x))

sech(x::DoubleFloat{T}) where {T<:IEEEFloat} =
    inv(cosh(x))

coth(x::DoubleFloat{T}) where {T<:IEEEFloat} =
    inv(tanh(x))