module MatrixDecomposition
export luls, lupp, choleskyls, qrls, lulspp

# Solve a system of linear equations using backwards substitution
function bwsub(a::Array{Float64, 2}, v::Array{Float64, 1})
	# Make sure we have an upper triangular matrix
	if !istriu(a)
		error("Matrix is not upper triangular")
	end

	# Create result vector the same size as v
	result = zeros(v)

	# The last value in result = last value in v scaled by last value in a
	result[end] = v[end] / a[end]

	# Iterate backwards through every row of a (except the last)
	for i in size(a, 1)-1:-1:1
		# Perform backwards substitution at each step
		sumterm = sum([a[i,j] * result[j] for j in i+1:size(a, 1)])
		result[i] = (1.0 / a[i,i]) * (v[i] - sumterm)
	end

	return result
end

# Solve a system of linear equations using forwards substitution
function fwsub(a::Array{Float64, 2}, v::Array{Float64, 1})
	# Make sure we have a lower triangular matrix
	if !istril(a)
		error("Matrix is not lower triangular")
	end

	# Create result vector the same size as v
	result = zeros(v)

	# The first value in result = the first value in v scaled by first value in a
	result[1] = v[1] / a[1]

	# Iterate forward through every row of a (except the first)
	for i in 2:size(a, 1)
		# Perform forwards substitution at each step
		sumterm = sum([a[i,j] * result[j] for j in 1:i-1])
		result[i] = (1.0 / a[i,i]) * (v[i] - sumterm)
	end

	return result
end

# Swap two rows of a matrix (works for any number filled matrix)
function swaprows!{T<:Number}(a::Array{T, 2}, i::Int64, j::Int64)
	# For every number across the first row
	for k in 1:size(a, 2)
		# Swap with the corresponding number in the second row
		a[i,k], a[j,k] = a[j,k], a[i,k]
	end
end

# Return stricly lower triangular portion of matrix
function stricttril{T<:Number}(a::Array{T, 2})
	return tril(a) - diagm(diag(a))
end

# Find LU Decomposition of a matrix using Crout's algorithm
function crout{T<:Number}(a::Array{T, 2}; print=false)
	N = size(a, 2)

	# Create diagonal NxN matrix to hold result in place
	result = copy(a)

	# Create matrix to store our permutations
	permuts = int(eye(a))

	# Perform Crout's algorithm in place
	for p in 1:N
		# Find u's
		for j in p:N
			# If j = 1 the sum term is taken to be zero
			sumterm = j == 1 ? 0 : sum([result[p,k] * result[k,j] for k in 1:p-1])

			result[p,j] = result[p,j] - sumterm
		end

		# Find v's
		for i in p+1:N
			# If i = 1 the sum term is taken to be zero
			sumterm = i == 1 ? 0 : sum([result[i,k] * result[k,p] for k in 1:p-1])
			result[i,p] = (result[i,p] - sumterm) / result[p,p]
		end
	end

	# Extract l and u from result matrix
	u = triu(result)
	l = stricttril(result) + eye(result)

	# Print decomposition, if requested
	if print
		println("L:\n", l)
		println("U:\n", u)
		println()
	end

	return l, u
end

# Calculate least squares estimate using LU Decomposition via Crout's Algorithm
function luls{T<:Number}(x::Array{T, 2}, y::Array{T, 1}; print=false)
	# Calculate normal equations
	m = x' * x
	v = x' * y

	# Calculate LU decomposition
	l, u = crout(m, print=print)

	# Solve V = L * gamma using forward substitution
	gamma = fwsub(l, v)

	# Solve gamma = U beta
	beta = bwsub(u, gamma)

	return beta
end

# LU Decomposition with partial pivoting
# Returns L, U and the permutation matrix recording the row pivots
function lupp{T<:Number}(a::Array{T, 2}; print=false)
	N = size(a, 2)

	# Create a copy of the input to work on
	acopy = copy(a)

	# Maintain a record of our permutations
	permut = eye(a)

	for p in 1:N-1
		if print
			println("Pivot ", p)
		end

		# Find the largest value in the current column
		istar = indmax(abs(acopy[:,p]))

		# Swap rows in the permutation and working matrices
		if istar > p
			swaprows!(permut, istar, p)
			swaprows!(acopy, istar, p)
		end

		# Dividing by zero is bad
		if a[p,p] != 0
			# Calculate l's in column
			acopy[p+1:end,p] /= acopy[p,p]

			# Subtract product of row and column k from submatrix
			acopy[p+1:end,p+1:end] -= (acopy[p+1:end,p] * acopy[p,p+1:end])
		end

		if print
			println(acopy, "\n")
		end
	end

	u = triu(acopy)
	l = stricttril(acopy) + eye(acopy)

	if print
		println("L:\n", l)
		println("U:\n", u)
		println()
	end

	return l, u, permut
end

# Calculate least squares estimate using LU Decomposition with partial pivoting
function lulspp{T<:Number}(x::Array{T, 2}, y::Array{T, 1}; print=false)
	# Calculate normal equations
	m = x' * x
	v = x' * y

	# Calculate LU decomposition
	l, u, p = lupp(m, print=print)

	# Solve V = l * gamma using forward substitution
	gamma = fwsub(l, p' * v)

	# Solve gamma = U beta
	beta = bwsub(u, gamma)

	return beta
end

# Find Cholesky decomposition of a positive definite symmetric matrix
function cholesky{T<:Number}(a::Array{T, 2}; print=false)
	# Verify that matrix is positive definite and symmetric
	if !isposdef(a)
		error("Matrix is not positive definite")
	end

	if !issym(a)
		error("Matrix is not symmetric")
	end

	N = size(a, 2)

	# Create result matrix by copying lower triangle of a
	result = tril(a)

	# Iterate through lower diagonal cells
	for p in 1:N
		# Sum is taken to be zero if p=1
		sumterm = p == 1 ? 0 : sum([result[p,j]^2 for j in 1:p-1])
		result[p,p] = sqrt((result[p,p] - sumterm))

		for i in p+1:N
			sumterm = p == 1 ? 0 : sum([result[i,j] * result[p,j] for j in 1:p-1])
			result[i,p] = (result[i,p] - sumterm) / result[p,p]
		end
	end

	# Print decomposition, if requested
	if print
		println("L:\n", result)
		println()
	end

	return result
end

# Solve a system of linear equations via Cholesky decomposition
function solvecholesky{T<:Number}(a::Array{T, 2}, v::Array{T, 1}; print=false)
	# Compute Cholesky decomposition
	l = cholesky(a, print=print)

	# Solve Ly = b using forward substitution
	y = fwsub(l, v)

	# Solve L'x=y using backward substitution
	x = bwsub(l', y)

	return x
end

# Calculate least squares estimate using Cholesky decomposition
function choleskyls{T<:Number}(x::Array{T, 2}, y::Array{T,1}; print=false)
	# Form components of normal equations
	m = x' * x
	v = x' * y

	# Solve using Cholesky decomposition
	return solvecholesky(m, v, print=print)
end

# Project one vector onto another
function project{T<:Number}(u::Array{T, 1}, v::Array{T, 1})
	return (dot(u, v) / dot(u, u)) * u
end

# Orthonormalize a set of vectors using Gram-Schmidt
function gs{T<:Number}(a::Array{T, 2})
	N = size(a, 2)

	result = copy(a)

	# For each column vector in a
	for i in 1:N
		for j in 1:i-1
			# Subtract all the previous projections
			result[:,i] -= project(result[:,j], a[:,i])
		end
	end

	# Normalize columns
	denominators = sqrt(sum(result.^2.0, 1))
	result = result ./ denominators

	return result
end

# Find QR Decomposition using Gram-Schmidt
function qrgs{T<:Number}(a::Array{T, 2}; print=false)
	N = size(a, 2)

	# Make sure matrix is full rank
	if rank(a) != N
		error("Matrix is not full rank")
	end

	# Perform Gram-Schmidt orthogonalization
	q = gs(a)
	# Need to keep only the upper triangular due to rounding errors
	r = triu(q' * a)

	# Print decomposition if requested
	if print
		println("Q:\n", q)
		println("R:\n", r)
		println()
	end

	return q, r
end

# Calculate least squares estimates using QR Decomposition
function qrls{T<:Number}(x::Array{T, 2}, y::Array{T, 1}; print=false)
	# Form components of normal equations
	# Note - I think decomposing X'X instead of X leads to stability problems
	# but I did it this way to fulfill the parameters of an assignment.
	m = x' * x
	v = x' * y

	# Compute QR decomposition
	q, r = qrgs(m, print=print)

	# Solve Rbeta = Q'V using backward substitution
	return bwsub(r, q' * v)
end
end


### A Pluto.jl notebook ###
# v0.16.1

using Markdown
using InteractiveUtils

# This Pluto notebook uses @bind for interactivity. When running this notebook outside of Pluto, the following 'mock version' of @bind gives bound variables a default value (instead of an error).
macro bind(def, element)
    quote
        local el = $(esc(element))
        global $(esc(def)) = Core.applicable(Base.get, el) ? Base.get(el) : missing
        el
    end
end

# ╔═╡ b6ff5052-04ec-11eb-043b-458c42b19edb
begin
    import Pkg
    # activate the shared project environment
    Pkg.activate(Base.current_project())

	using Distributions
	using Plots
	using PlutoUI
	using Random
end

# ╔═╡ 87c1dc3b-d361-4f46-a68a-c28d7c7de0c3
md"""
# Equilibrium binding simulations

This notebook can either:

1. simulate the outcome of an equilibrium binding experiment given a hypothetical $K_D$ and desired experimental parameters (section [Simulate a binding curve](#d50721c7-34e4-411d-ae6b-dc6196b0f5a9)) or
2. calculate the fractional saturation of receptor given a $K_D$ and total concentrations of receptor and ligand (section [Predict saturation of receptor](#3954adfa-73cc-495a-93c1-bcde87a60ce8)).

The following reference explains very well the theory of equilibrium binding experiments, as well as many important practical considerations:

> Jarmoskaite I, AlSadhan I, Vaidyanathan PP & Herschlag D (2020) How to measure and evaluate binding affinities. *eLife* **9**: e57264 <https://doi.org/10.7554/eLife.57264>
"""

# ╔═╡ d50721c7-34e4-411d-ae6b-dc6196b0f5a9
md"""
## Simulate a binding curve

Enter values of $K_D$ and experimental parameters below to simulate the resulting data points (sampling the binding curve) as they would come out if you performed the experiment. The vertical line on the graph shows the value of $K_D$. The horizontal lines show zero and full saturation of the receptor.
"""

# ╔═╡ c4d66e7a-38e4-497d-b680-8c50c51fe146
md"""
Value of $K_D$: $(@bind kd PlutoUI.Slider(0.1:0.1:100.0, default = 10.0, show_value = true))

Concentration of ligand at highest titration point $L_{max}$: $(@bind lmax PlutoUI.Slider(kd:1.0:100000.0, default = 1000.0, show_value = true))

Dilution factor between titration points: $(@bind dilution PlutoUI.NumberField(1.1:0.1:10.0, default = 2.0))

Number of titration points: $(@bind points PlutoUI.NumberField(1:1:24, default = 15))

Concentration of receptor $R_{tot}$: $(@bind Rtot PlutoUI.Slider(0.001:1.0:50.0, default = 1.0, show_value = true))

Simulate random, normally distributed noise? $(@bind simulateNoise PlutoUI.CheckBox(default = false))
"""

# ╔═╡ b05df20b-353e-462d-aa13-534862f8f6cd
if simulateNoise
	md"""
	Mean (μ) and standard deviation (σ)

	μ = $(@bind μ PlutoUI.NumberField(-3.0:0.1:3.0, default = 0.0))
	σ = $(@bind σ PlutoUI.NumberField(0.0:0.001:0.1, default = 0.001))
	"""
end

# ╔═╡ d49dee36-c12d-4206-9699-a181570caa80
md"""
Ratio $\frac{L_{max}}{K_D} =$ $(lmax / kd)

Ratio $\frac{K_D}{R_{tot}} =$ $(kd / Rtot)
"""

# ╔═╡ 915b2a49-cb32-40a0-bfae-21380144ed3a
if simulateNoise
	md"""
	Signal-to-noise ratio $\frac{S}{σ} =$ $(1.0 / σ)
	"""
end

# ╔═╡ 3954adfa-73cc-495a-93c1-bcde87a60ce8
md"""
## Predict saturation of receptor

Value of $K_D$: $(@bind kd2 PlutoUI.NumberField(1.0:10.0:1000.0, default = 10.0))

Concentration of ligand $L_{tot}$: $(@bind ltot2 PlutoUI.NumberField(1.0:100.0:100000.0, default = 1000.0))

Concentration of receptor $R_{tot}$: $(@bind rtot2 PlutoUI.NumberField(0.001:0.01:100000.0, default = 1000.0))

Percent saturation of receptor $B =$
"""

# ╔═╡ ae1d6dac-92d7-4962-a641-63965f73a8f2
md"""
## Code

The quadratic model gives $B$, the fraction of bound receptor (or fractional saturation of receptor) for any values of receptor concentration $R_{tot}$ and ligand concentration $L_{tot}$, given a binding affinity determined by the value of $K_D$:

$B = \frac{(K_{D} + R_{tot} + L_{tot}) - \sqrt{(- K_{D} - R_{tot} - L_{tot})^2 - 4 \times R_{tot} \times L_{tot}}}{2 \times R_{tot}}$

The function below defines this binding model in code:
"""

# ╔═╡ 1eed6126-331b-4218-acc7-8b7c132ef355
function quadratic(Ltot)
	( (kd + Rtot + Ltot) - sqrt((- kd - Rtot - Ltot) ^ 2 - 4 * Rtot * Ltot) ) / (2 * Rtot)
end

# ╔═╡ a5ac735b-f748-4d37-9cf1-7e64e1ea4c53
function quadratic(kd, Rtot, Ltot)
	( (kd + Rtot + Ltot) - sqrt((kd + Rtot + Ltot) ^ 2 - 4 * Rtot * Ltot) ) / (2 * Rtot)
end

# ╔═╡ 4d816572-03a3-481c-a29c-522f0535a679
round(100 * quadratic(kd2, ltot2, rtot2), digits = 2)

# ╔═╡ 3794717b-1f61-4622-9a55-ad86ab947831
md"""
The code below calculates concentrations of the titration series ($X$ values for the plot) given the maximal ligand concentration $L_{max}$, the dilution factor and the number of titration points entered above:
"""

# ╔═╡ 7c43edeb-a5e6-4fb4-8494-e2072d0d56fd
x = [ lmax / dilution^i for i in 0:points-1 ]

# ╔═╡ 06f341e8-6d40-4eb7-b7e9-58c5d45f1f85
md"""
The code below uses the quadratic model to calculate fractional saturation of the receptor ($Y$ values for the plot) at each titration point for the values of $K_D$ and $R_{tot}$ entered above:
"""

# ╔═╡ 82af41ba-d9b9-4ce2-8736-6fe32c8e024e
if simulateNoise
	noise = rand(Normal(μ, σ), points)
	y = quadratic.(x) .+ noise
else
	y = quadratic.(x)
end

# ╔═╡ efc70ae1-abcc-472d-8df8-c55cbbd6529b
begin
	scatter(
		x,
		y,
		xscale = :log10,
		legend = :none,
		xlabel = "Ligand concentration",
		ylabel = "Predicted fractional saturation of receptor"
	)
	vline!([kd], color = :red)
	hline!([1.0], color = :black)
	hline!([0.0], color = :black)
end

# ╔═╡ 9c5021f3-67ab-464d-8f22-25e269c36083
md"""
## Required packages
"""

# ╔═╡ b0340ca3-02d5-49a9-841b-7827e4aee2bb
PlutoUI.TableOfContents()

# ╔═╡ Cell order:
# ╟─87c1dc3b-d361-4f46-a68a-c28d7c7de0c3
# ╟─d50721c7-34e4-411d-ae6b-dc6196b0f5a9
# ╟─c4d66e7a-38e4-497d-b680-8c50c51fe146
# ╟─b05df20b-353e-462d-aa13-534862f8f6cd
# ╟─d49dee36-c12d-4206-9699-a181570caa80
# ╟─915b2a49-cb32-40a0-bfae-21380144ed3a
# ╟─efc70ae1-abcc-472d-8df8-c55cbbd6529b
# ╟─3954adfa-73cc-495a-93c1-bcde87a60ce8
# ╠═4d816572-03a3-481c-a29c-522f0535a679
# ╟─ae1d6dac-92d7-4962-a641-63965f73a8f2
# ╠═1eed6126-331b-4218-acc7-8b7c132ef355
# ╠═a5ac735b-f748-4d37-9cf1-7e64e1ea4c53
# ╟─3794717b-1f61-4622-9a55-ad86ab947831
# ╠═7c43edeb-a5e6-4fb4-8494-e2072d0d56fd
# ╟─06f341e8-6d40-4eb7-b7e9-58c5d45f1f85
# ╠═82af41ba-d9b9-4ce2-8736-6fe32c8e024e
# ╟─9c5021f3-67ab-464d-8f22-25e269c36083
# ╠═b6ff5052-04ec-11eb-043b-458c42b19edb
# ╠═b0340ca3-02d5-49a9-841b-7827e4aee2bb


"""
# module FiniteDifference

# Examples

```jldoctest
julia>
```
"""
module FiniteDifference

using BandedMatrices

export forward_difference_matrix,
    backward_difference_matrix,
    cyclic_difference_matrix,
    centered_difference_matrix

"""
    forward_difference_matrix(T::Type{<:Number}, shape::NTuple{N,<:Number})

Create a forward difference matrix.

The default type of returned value `A` is a `BandedMatrix`.
If you want a regular `Matrix`, run `Matrix(A)`.
"""
function forward_difference_matrix(T::Type{<:Number}, shape::NTuple{N,<:Number}) where {N}
    A = BandedMatrix(-Ones{T}(shape), (0, 1))
    A[band(1)] .= 1
    return A
end

"""
    backward_difference_matrix(T::Type{<:Number}, shape::NTuple{N,<:Number})

Create a backward difference matrix.

The default type of returned value `A` is a `BandedMatrix`.
If you want a regular `Matrix`, run `Matrix(A)`.
"""
function backward_difference_matrix(T::Type{<:Number}, shape::NTuple{N,<:Number}) where {N}  # I am unsure about this construction.
    A = BandedMatrix(Ones{T}(shape), (1, 0))
    A[band(-1)] .= -1
    return A
end

function cyclic_difference_matrix(T::Type{<:Number}, shape::NTuple{N,<:Number}) where {N}
    A = BandedMatrix(Ones{T}(shape), (1, 0))
    A[band(-1)] .= -1
    B = Matrix(A)
    B[1, end] = -1
    return B
end

function centered_difference_matrix(T::Type{<:Number}, shape::NTuple{N,<:Number}) where {N}
    A = BandedMatrix(Zeros{T}(shape), (1, 1))
    A[band(-1)] .= -1
    A[band(1)] .= 1
    return A
end

end


# Elementary row operations

export row_swap!, row_scale!, row_add_mult!

"""
`row_swap!(A,i,j)` swaps rows `i` and `j` in the matrix `A`.
"""
function row_swap!(A::AbstractMatrix{T}, i::Int, j::Int) where {T}
    r, c = size(A)
    @assert (1 <= i <= r) && (1 <= j <= r) "Row index out of bounds"
    if i == j
        return
    end

    save_row = [A[i, k] for k = 1:c]

    for k = 1:c
        @inbounds A[i, k] = A[j, k]
    end

    for k = 1:c
        @inbounds A[j, k] = save_row[k]
    end
end

"""
`row_scale!(A,i,s)` multiplies all entries in row `i`
of `A` by `s`.
"""
function row_scale!(A::AbstractMatrix{T}, i::Int, s) where {T}
    r, c = size(A)
    @assert 1 <= i <= r "Row index out of bounds"

    for k = 1:c
        @inbounds A[i, k] = s * A[i, k]
    end
end

"""
`row_add_mult!(A,i,s,j)` adds `s` times row `i` to row `j`
in the matrix `A`.
"""
function row_add_mult!(A::AbstractMatrix{T}, i::Int, s, j::Int) where {T}
    r, c = size(A)
    @assert (1 <= i <= r) && (1 <= j <= r) "Row index out of bounds"

    save_row = [s * A[i, k] for k = 1:c]
    for k = 1:c
        @inbounds A[j, k] = A[j, k] + save_row[k]
    end
end


# GSA code v0.1.
# Coded by Jesús Mejía.
# Based on MATLAB code of Esmat Rashedi, 2010.
# " E. Rashedi, H. Nezamabadi-pour and S. Saryazdi,
# “GSA: A Gravitational Search Algorithm”, Information sciences, vol. 179,
# no. 13, pp. 2232-2248, 2009."

function massCalculation(fit,searchType)
	####here, make your own function of 'mass calculation'

	Fmax = maximum(fit)
	Fmin = minimum(fit)
	Fmean= mean(fit)
	N = length(fit)

	if Fmax == Fmin
		M = ones(N)
	else

		if searchType == :minimize #for minimization
			best = Fmin
			worst= Fmax #eq.17-18.
		else #for maximization
			best = Fmax
			worst= Fmin #eq.19-20.
		end

		M = (fit .- worst) ./ (best .- worst) #eq.15,

	end

	M = M ./ sum(M) #eq. 16.

	return M

end

function Gconstant(iteration,max_it)
	# here, make your own function of 'G'
	α = 20
	G0= 100

	G = G0*exp(-α * iteration / max_it) #eq. 28.

	return G
end

#This function calculates the accelaration of each agent in gravitational field. eq.7-10,21.
function Gfield(M,X,G,Rnorm,Rpower,ElitistCheck,iteration,max_it)
	N, D   = size(X)
	final_per= 2 #In the last iteration, only 2 percent of agents apply force to the others.

	####total force calculation
	if ElitistCheck == 1
		kbest = final_per+(1-iteration/max_it)*(100-final_per) #kbest in eq. 21.
		kbest = round(Int, N*kbest/100)
	else
		kbest = N #eq.9.
	end

	ds = sortperm(M, rev=true)

	E = zeros(N, D)
	for i=1:N
		for ii=1:kbest
			j = ds[ii]
			if j != i
				R = norm(X[i,:]-X[j,:], Rnorm) #Euclidian distanse.
				for k=1:D
                    E[i, k]= E[i,k]+rand() * (M[j]) * ((X[j,k]-X[i,k])/(R^Rpower + eps()))
					#note that Mp(i)/Mi(i)=1
				end
			end
		end
	end

	##acceleration
	a = E.*G #note that Mp(i)/Mi(i)=1

	return a
end

# This function updates the velocity and position of agents.
function move(X,a,V)
	# movement.
	N, D = size(X)
	V = rand(N, D).*V + a # eq. 11.
	X = X + V # eq. 12.

	return X, V
end


export proxNuclear!, normNuclear

"""
    proxNuclear!(x::Vector{T}, λ::Float64; svtShape::NTuple=[],kargs...)

applies singular value soft-thresholding - i.e. the proximal map for the nuclear norm regularization.

# Arguments:
* `x::Array{T}`          - Vector to apply proximal map to
* `λ::Float64`           - regularization paramter
* `svtShape::NTuple=[]`  - size of the underlying matrix
"""
function proxNuclear!(x::Vector{T}, λ::Float64; svtShape::NTuple=[],kargs...) where T
  U,S,V = svd(reshape(x, svtShape))
  proxL1!(S,λ)
  x[:] = vec(U*Matrix(Diagonal(S))*V')
end

"""
    normNuclear(x::Vector{T}, λ::Float64; svtShape::NTuple=[],kargs...) where T

returns the value of the nuclear norm regularization term.
Arguments are the same as in `proxNuclear!`
"""
function normNuclear(x::Vector{T}, λ::Float64; svtShape::NTuple=[],kargs...) where T
  U,S,V = svd( reshape(x, svtShape) )
  return λ*norm(S,1)
end


using SpecialFunctions

export hann_window, rect_window, kaiser_window, tri_window, lanczos_window

"""
    rect_window(width = 0.5)

Rectangular window which is 1 from 0 to `width` and 0 otherwise.

Can be used to calculate a moving average, for example.
"""
rect_window(width = 0.5) = WindowFunction(x -> 1.0, width)

"""
    hann_window(width = 1)

Hann window (raised cosine) that is 1 at 0 and reaches 0 at `width`.
"""
hann_window(width = 1.0) = WindowFunction(
    x -> cos(pi * x / 2 / width)^2,
    width
)

"""
    kaiser_window(width = 1.2; alpha = 3.5)

Kaiser window cut off at `width` with the shape parameter `alpha`.
"""
kaiser_window(width = 1.2; alpha = 3.5) = WindowFunction(
    x -> besseli(0, pi * alpha * sqrt(1 - (x / width)^2)) / besseli(0, pi * alpha),
    width
)

"""
    tri_window(width = 1)

Triangular window which is 1 at 0 and reaches 0 at `width`.

Can be used to perform linear interpolation when used as upsampling_function
with `width=1`.
"""
tri_window(width=1.) = WindowFunction(
    x -> 1 - x/width,
    width
)

"""
    lanczos_window(lobes = 3; width = 1)

Lanczos window which is 1 at 0 and 0 at 1*`width`, 2*`width`, 3*`width`, ...

The higher the number of lobes, the higher the accuracy of sinc approximation.
"""
function lanczos_window(lobes::Int = 3; width=1.)
    @assert lobes > 0 "Lanczos window is only defined for at least one lobe"

    WindowFunction(
    x -> begin
        x == 0 && return 1.
        x = x/width
        lobes*sin(pi*x)*sin(pi*x/lobes)/(pi^2*x^2)
    end,
    lobes*width
    )
end


using Test
using CompScienceMeshes

sp = CompScienceMeshes.SphereChart(point(1,2,3), 1.0)
for L in 0:10
    @eval @test sum(w for (p,w) in quadpoints(sp, $L)) ≈ 4π
end

function Ylm(K, r)

    @assert K >= 2 "Multipole order needs to be equal or larger than 2."

    r = r / norm(r)
    x = r[1]; y = r[2]; z = r[3]

    T = eltype(r)
    Y = zeros(Complex{T}, K*K)

    Y[1] = 1/√(4π)

    facphip =  x + im*y;
    facphim = -x + im*y;
    for m ∈ 1:(K-1)
        Y[(m+1)*(m+1)] = -Y[m*m] * facphip * sqrt((m+0.5)/m)
        Y[m*m+1] = -Y[(m-1)*(m-1)+1] * facphim * sqrt((m+0.5)/m)
    end

    Y[3] = √3 * z * Y[1]
    for l ∈ 2:(K-1)
        Y[(l+1)*(l+1)-1] = sqrt(2l+1) * z * Y[l*l]
        Y[l*l+2] = sqrt(2*l+1) * z * Y[(l-1)*(l-1)+1]
    end

    for l ∈ 2:(K-1)
        for m ∈ (2-l):(l-2)
            Y[l*(l+1)+m+1] =
                sqrt((4*l^2-1)/(l^2-m^2)) * z * Y[l*(l-1)+m+1] -
                sqrt(((2*l+1)*((l-1)^2-m^2))/((l^2-m^2)*(2*l-3))) * Y[(l-1)*(l-2)+m+1]
        end
    end

    Y
end

K = 3
I = zeros(K^2)
for (_p,w) in quadpoints(sp, 20)
    Y = Ylm(K, cartesian(_p))
    for l in 0:K-1
        for _m in -l:l
            i = l*(l+1)+_m+1
            I[i] += w * real(Y[i] * conj(Y[i]))
        end
    end
end

@test all(isapprox.(I, 1.0))


using Distributions, StatsBase, HypothesisTests, Plots, Random; pyplot()
Random.seed!(0)

n = 25
N = 10^4
xGrid = -10:0.001:10
kGrid = 0:0.01:5
dist1, dist2 = Exponential(1), Normal()

function ksStat(dist)
    data = rand(dist,n)
    Fhat = ecdf(data)
    sqrt(n)*maximum(abs.(Fhat.(xGrid) - cdf.(dist,xGrid)))
end

kStats1 = [ksStat(dist1) for _ in 1:N]
kStats2 = [ksStat(dist2) for _ in 1:N]

p1 = stephist(kStats1, bins=50,
	c=:blue, label="KS stat (Exponential)", normed=true)
p1 = plot!(kGrid, pdf.(Kolmogorov(),kGrid),
	c=:red, label="Kolmogorov PDF", xlabel="K", ylabel="Density")

p2 = stephist(kStats2, bins=50,
	c=:blue, label="KS stat (Normal)", normed=true)
p2 = plot!(kGrid, pdf.(Kolmogorov(),kGrid),
	c=:red, label="Kolmogorov PDF", xlabel="K", ylabel="Density")

plot(p1, p2, xlims=(0,2.5), ylims=(0,1.8), size=(800, 400))


module Bresenham

function line(f, x0::Int, y0::Int, x1::Int, y1::Int)
  dx = 2 * (x1 - x0)
  dy = 2 * (y1 - y0)

  if dx != 0
    if dy != 0
      if abs(dx) >= abs(dy)
        e = dx / 2
        y = y0
        for x = x0:sign(dx):x1
          f(x, y)
          # (a) following code runs even if
          # the loop exits after this iteration
          # but we don't care about that side effect
          e -= sign(dx) * sign(dy) * dy
          if sign(dx) > 0 && e < 0 ||
             sign(dx) < 0 && e >= 0
            y += sign(dy)
            e += dx
          end
        end
      else # abs(dx) < abs(dy) (steep case)
        e = dy / 2
        x = x0
        for y = y0:sign(dy):y1
          f(x, y)
          # see (a)
          e -= sign(dx) * sign(dy) * dx
          if sign(dy) > 0 && e < 0 ||
             sign(dy) < 0 && e >= 0
            x += sign(dx)
            e += dy
          end
        end
      end
    else # dy == 0 (horizontal line)
      for x = x0:sign(dx):x1
        f(x, y0)
      end
    end
  else # dx == 0 (vertical line)
    for y = y0:sign(dy):y1
      f(x0, y)
    end
  end
end # function

function circle(f, x0::Int, y0::Int, r::Int)
  x = r - 1
  y = 0
  dx = 1
  dy = 1
  err = dx - (r << 1)

  while x >= y
    f(x0 + x, y0 + y)
    f(x0 + y, y0 + x)
    f(x0 - y, y0 + x)
    f(x0 - x, y0 + y)
    f(x0 - x, y0 - y)
    f(x0 - y, y0 - x)
    f(x0 + y, y0 - x)
    f(x0 + x, y0 - y)

    if err <= 0
      y += 1
      err += dy
      dy += 2
    end

    if err > 0
      x -= 1
      dx += 2
      err += dx - (r << 1)
    end
  end
end # function

function ordered_circle(f, x0::Int, y0::Int, r::Int)
  pts = Vector{Tuple{Int, Int}}()
  Bresenham.circle(x0, y0, r) do x, y
    push!(pts, (x, y))
  end

  unique!(sort!(pts, by=(X -> atan(X[2], X[1]))))
  for X in pts
    f(X...)
  end
end # function

export line, circle, ordered_circle

end # module


geosum(r::Real, n::Integer) = (1 - r^(n + 1)) / (1 - r)
geosum(a::Real, r::Real, n::Integer) = a * geosum(r, n)

monthly_rate(annual_rate::Real) = (1 + annual_rate) ^ (1/12) - 1

"""
Future value. Assumes payments at the beginning of the month.
"""
function fv(monthly_payment::Real, annual_rate::Real; months::Integer=0,
            init::Real=0, years::Integer=0)
  months == 0 && (months = years * 12)

  r = monthly_rate(annual_rate)
  init_future = init * (1 + r) ^ months
  return geosum(monthly_payment, 1 + r, months) - monthly_payment + init_future
end

"""
Computes monthly payments. Assumes payments at the beginning of the month.
"""
function monthly_payment(goal::Real, annual_rate::Real; months::Integer=0,
                         years::Integer=0, init::Real=0)
  months == 0 && (months = years * 12)
  r = monthly_rate(annual_rate)
  init_future = init * (1 + r) ^ months
  return (goal - init_future) / (geosum(1 + r, months) - 1)
end


# Example of Bayesian estimation
# sampling from exponential(theta) distribution
# lognormal prior

# Shows how likelihood, joint, marginal likelihood,
# and posterior are formed, and how posterior can be
# integrated to get posterior mean

# explore different sample sizes, different true thetas
using Distributions, Plots

function main()
    n = 50   # sample size
    theta = 3 # true theta
    y = rand(Exponential(theta), n) # sample from exponential(theta)
    # make plots
    thetas = range(0.01,stop=10,length=1000)
    delta = thetas[2]-thetas[1]
    p, priormean = prior(thetas)
    post, postmean = posterior(y, thetas)
    plot(thetas, [p post], label = ["prior" "posterior"])
    plot!([priormean, priormean], [0.0, 1.0], label = "prior mean")
    plot!([postmean, postmean], [0.0, 1.0], label = "posterior mean")

end

# the prior is lognormal(1,1)
function prior(theta)
    d = LogNormal(1.0,1.0)
    p = pdf.(Ref(d), theta)
    pmean = exp(1.5) # mean of lognormal is exp(mu+sig/2)
    return p, pmean
end

# the likelihood function
function likelihood(y, theta)
    dens = zeros(size(theta))
    for i = 1:size(theta,1)
        d = Exponential(theta[i])
        dens[i] = prod(pdf.(Ref(d), y))
    end
    return dens
end

# joint is prior X likelihood
function joint(y, theta)
    l = likelihood(y, theta)
    p, junk = prior(theta)
    dens = l.*p
end

# compute marginal likelihood of Y by integrating out theta (crude, only illustrative)
function marginal(y)
    dens = 0.0
    theta = 0.0
    delta = 0.01
    # evaluate joint over grid
    for r = 1:1000
        theta += delta
        dens += joint(y, theta)
    end
    # marginalize by integrating the joint (sum up height X width)
    dens = dens*delta
end

# the posterior, by Bayes' Law
function posterior(y, theta)
    m = marginal(y)
    j = joint(y, theta)
    dens = j ./ m
    thetas = range(0.01,stop=10,length=1000)
    pmean = sum(dens.*thetas.*0.01)
    return dens, pmean
end

main()


export fourier_points, fourier_wavenumbers
using FFTW
"""
fourier_nodes(n; a = 0, b = 2π)

# Description
- Create a uniform grid of points for periodic functions

# Arguments
- `N`: integer | number of evenly spaced points

# Keyword Arguments
- `a`: number | starting point of interval [a, b)
- `b`: number | ending point of interval [a, b)

# Return
- `g`: array | an array of points of evenly spaced points from [a, b)
"""
function fourier_nodes(N; a = 0, b = 2π)
    return (b-a) .* collect(0:(N-1))/N .+ a
end

"""
fourier_wavenumbers(N; L = 2π)

# Description
- Create wavenumbers associated with the domain of length L

# Arguments
- `N`: integer | number of wavevectors

# Keyword Arguments
- `L`: number | length of interval [a, b), L = b-a

# Return
- `wavenumbers`: array | an array of wavevectors
"""
function fourier_wavenumbers(N; L = 2π)
    up = collect(0:1:N-1)
    down = collect(-N:1:-1)
    indices = up
    indices[div(N,2)+1:end] = down[div(N,2)+1:end]
    wavenumbers = 2π / L .* indices
    return wavenumbers
end


"""
A simple implementation of calculating edit distance of two strings
"""
function edit_distance(s1::ASCIIString, s2::ASCIIString)
    m=length(s1)+1
    n=length(s2)+1
    i=0
    j=0
    tbl = Array{Int64}(m,n)
    for i=1:m
        tbl[i,1]=i-1
    end
    for j=1:n
        tbl[1,j]=j-1
    end
    for i=2:m
        for j=2:n
            cost = s1[i-1] == s2[j-1]?0:1
            tbl[i,j] = min(tbl[i, j-1]+1, tbl[i-1, j]+1, tbl[i-1, j-1]+cost)
        end
    end
    return tbl[i,j]
end

function hamming_distance(s1::ASCIIString, s2::ASCIIString)
    d = 0
    len = min(length(s1), length(s2))
    for i in 1:len
        if s1[i] != s2[i]
            d += 1
        end
    end
    return d
end

function edit_distance(s1::Sequence, s2::Sequence)
    return edit_distance(s1.seq, s2.seq)
end

function hamming_distance(s1::Sequence, s2::Sequence)
    return hamming_distance(s1.seq, s2.seq)
end


"""
    subset_sum_test(array::Array{Int64,1}, target::Int64)

The recursive expression tests if there is a subset in the array, where the sum of the
subset is equal to the `target`.

# Arguments
- `array::Array{Int64,1}`: Array with a possible sum of a subsequence.
- `target::Int64`: Target sum of the subsequence, which should be included in the array.

# Examples
```julia-repl
julia> import ClassicAlgorithmsCollections
julia> set = [3, 34, 4, 12, 5, 2]
julia> sum = 9
julia> ClassicAlgorithmsCollections.subset_sum_test(set, sum)
true
```
"""
function subset_sum_test(array::Array{Int64,1}, target::Int64)
    n = length(array)
    if target == 1
        return true
    end
    if n == 1
        return false
    end
    if array[n-1] > target
        return subset_sum_test(array[1:n-1], target)
    end

    return subset_sum_test(array[1:n-1], target) ||
           subset_sum_test(array[1:n-1], target - array[n-1])
end


"""Julia program to implement Interpolation Search algorithm.
It is simliar to a person searching for a name in a telephone directory
The algorithm calculates where in the remaining array space the required item would be,
based on the values at the bounds of the search space and the required element, usually via a linear interpolation
"""

function interpolation_search(arr, n, ele)
    low = 1
    high = n
    while low <= high && ele >= arr[low] && ele <= arr[high]
        if(low == high)
            # If element found
            if(arr[low] == ele)
                return true
            end
            return false
        end
        # Formula to return the value of 'pos'
        pos = low + (((high - low) ÷ (arr[high] - arr[low]))* (ele - arr[low]))
        # If the pos element is the required element, return that index
        if(arr[pos] == ele)
            return true
        end
        # Else find the section of the array in which the required element belongs to
        if(arr[pos] < ele)
            low = pos + 1
        else
            high = pos - 1
        end
    end
    return false
end

print("How many numbers are present in the array? ")
n = readline()
n = parse(Int, n)
if (n <= 0)
    println("Array is Empty!!!")
    exit()
end
arr = Int[]
print("Enter the numbers: ")
arr = [parse(Int, num) for num in split(readline())]
print("Which number do you want to search in the array? ")
ele = readline()
ele = parse(Int, ele)
# Sort the array in ascending order
arr = sort(arr)
res = interpolation_search(arr, n, ele)
if (res == 0)
    print("The number $ele is not present in the array")
else
    print("The number $ele is present in the array.")
end

"""
Time Complexity - O(log(n)), where 'n' is the size of the array
Space Complexity - O(n)

SAMPLE INPUT AND OUTPUT

SAMPLE I

How many numbers are present in the array? 5
Enter the numbers: 1 2 3 4 5
Which number do you want to search in the array? 6
The number 6 is not present in the array

SAMPLE II

How many numbers are present in the array? 3
Enter the numbers: 3 1 2
Which number do you want to search in the array? 2
The number 2 is present in the array.

"""


#=
Example 2: Fit a linear model first, then add a further component and
fit a quadratic model.

This example is very similar to #1, but it shows how to check for
goodness of fit, and how to add a new component to obtain a better fit.
=#
using Random, DataFitting

# "True" parameter values:
p0 = 1.
p1 = 2.
p2 = 3.

# Evaluate the model with "True" parameters
x = 0.:0.1:10;
y = collect(p0 .+ p1 .* x .+ p2 .* x.^2);

# Add some noise to simulate a measurement process
noise = 1.
y .+= noise .* randn(length(x));

# Wrap empirical measures and uncertainties in a `Measure` object
data = Measures(y, noise)

# Prepare the model.  Note that we start with an empty model, and add
# the components one by one:
model = Model()
add_comp!(model, :p0 => 1)    # add 1st component
add_comp!(model, :p1 => 2)    # add 2nd component
add_dom!(model, x)                         # set domain
add_expr!(model, :(p0 .+ p1 .* domain[1])) # add model expression

# Fit the model to empirical data
result = fit!(model, data)

# Plot data and best fit model
using Gnuplot
@gp    domain(model) data.val data.unc "w yerr t 'Data'" :-
@gp :- domain(model) model() "w line t 'Model'"

# Print the reduced chi-squared and the probability that random noise
# may lead to higher chi-squared values.  Note that the latter is
# given as logarithmic value
println("Red. χ^2 = ", result.cost / result.dof)
println("Test probability = ", 10^result.log10testprob)

# Clearly this is not a good fit.  Let's add a 3rd `ScalarParam`
# component to obtain a quadratic model
add_comp!(model, :p2 => 3)          # add 3rd component

# Replace model expression
replaceexpr!(model, :(p0 .+ p1 .* domain[1] .+ p2 .* domain[1].^2))

# Fit model to empirical data and plot
result = fit!(model, data)
@gp    domain(model) data.val data.unc "w yerr t 'Data'" :-
@gp :- domain(model) model() "w line t 'Model'"

# Compare "true" parameter values with best fit ones:
using Printf
@printf "%f ∼ %f  (accuracy: %5.3f%%)" p0 result.p0.par.val 100. * (result.p0.par.val-p0) / p0
@printf "%f ∼ %f  (accuracy: %5.3f%%)" p1 result.p1.par.val 100. * (result.p1.par.val-p1) / p1
@printf "%f ∼ %f  (accuracy: %5.3f%%)" p2 result.p2.par.val 100. * (result.p2.par.val-p2) / p2


# detrend()
@testset "Test detrend()" begin
    x = 1:100
    y = sin.(0.4x) .+ cos.(x)

    # Offset
    offset = 10
    y_offset = y .+ offset
    y_detrended, coef = detrend(y_offset, p = 0, return_coefs = true)
    @test isapprox(y, y_detrended, atol = 0.5)
    @test isapprox(offset, coef, atol = 0.05)

    # Linear trend
    m = 0.5
    b = 5
    y_linear = y .+ (m*x .+ b)
    y_detrended, coefs = detrend(y_linear, p = 1, return_coefs = true)
    @test isapprox(y, y_detrended, atol = 0.5)
    @test isapprox(b, coefs[1], atol = 0.05)
    @test isapprox(m, coefs[2], atol = 0.05)

    # detrend by known coefs
    y_detrended2, coefs2 = detrend(y_linear, p = 1, coefs = coefs, return_coefs = true)
    @test all(y_detrended .== y_detrended2)
    @test all(coefs .== coefs2)

    # Errors
    @test_throws DomainError detrend(y_linear, p = -1, return_coefs = true)
    @test_throws DomainError detrend(y_linear, p = 0, coefs = coefs, return_coefs = true)
end

@testset "Theil Sen estimator" begin
    @test theilSenEstimator(collect(1:10), collect(1:10)) == (1,0)

    # Errors
    @test_throws DomainError theilSenEstimator([1,2], [3,4,5])

end


using LinearAlgebra
using SparseArrays
using Plots

"This code solves the 2D Laplace's equation with nonzero Dirichlet and Neumann
boundary conditions using a finite difference method."

m = 100 # number of points
x = LinRange(-1,1,m+2)
y = LinRange(-1,1,m+2)
xint = x[2:end-1]
yint = y[2:end-1]
h = x[2]-x[1]

# ordering of unknowns
# u14 u24 u34 u44
# u13 u23 u33 u43
# u12 u22 u32 u42
# u11 u21 u31 u41

id(i,j,m) = i + (j-1)*m # return global index from i,j
function FD_matrix_2D(h,m)
    A = spzeros(m*m,m*m)
    for i = 1:m, j = 1:m # loop thru indices
        A[id(i,j,m),id(i,j,m)] = 4/h^2
        if i > 1 # avoids leftmost line of nodes
            A[id(i,j,m),id(i-1,j,m)] = -1/h^2 # x-derivative
        end
        if i < m # avoids rightmost line of nodes
            A[id(i,j,m),id(i+1,j,m)] = -1/h^2
        end
        if j > 1
            A[id(i,j,m),id(i,j-1,m)] = -1/h^2 # y-derivative
        end
        if j < m
            A[id(i,j,m),id(i,j+1,m)] = -1/h^2
        end
    end
    return A
end

A = FD_matrix_2D(h,m)

f(x,y) = exp(-10*((x-.25)^2 + (y-.75)^2))
# f(x,y) = 1.0
b = vec(f.(xint,yint'))

# solve Au = f(xi,yj)
u = A\vec(b)
contourf(xint,yint,u)


"""
    poisson_time(a, b, u)

Obtaining waiting time for inhomogeneous Poisson Process
with rate of the form λ(t) = (a + b*t)^+, `a`,`b` ∈ R, `u` uniform random variable
"""
function poisson_time(a, b, u)
    if b > 0
        if a < 0
            return sqrt(-log(u)*2.0/b) - a/b
        else # a[i]>0
            return sqrt((a/b)^2 - log(u)*2.0/b) - a/b
        end
    elseif b == 0
        if a > 0
            return -log(u)/a
        else # a[i] <= 0
            return Inf
        end
    else # b[i] < 0
        if a <= 0
            return Inf
        elseif -log(u) <= -a^2/b + a^2/(2*b)
            return -sqrt((a/b)^2 - log(u)*2.0/b) - a/b
        else
            return Inf
        end
    end
end

"""
    poisson_time((a, b, c), u)

Obtaining waiting time for inhomogeneous Poisson Process
with rate of the form λ(t) = c + (a + b*t)^+,
where `c`> 0 ,`a, b` ∈ R, `u` uniform random variable
"""
function poisson_time((a,b,c)::NTuple{3}, u)
    if b > 0
        if a < 0
            if -c*a/b + log(u) < 0.0
                return sqrt(-2*b*log(u) + c^2 + 2*a*c)/b - (a+c)/b
            else
                return -log(u)/c
            end
        else # a >0
            return sqrt(-log(u)*2.0*b + (a+c)^2)/b - (a+c)/b
        end
    elseif b == 0
        if a > 0
            return -log(u)/(a + c)
        else # a <= 0
            return -log(u)/c
        end
    else # b < 0
        if a <= 0.0
            return -log(u)/c
        elseif  - c*a/b - a^2/(2*b)  + log(u) > 0.0
            return +sqrt((a+c)^2 - 2.0*log(u)*b)/b - (a+c)/b
        else
            return (-log(u)+ a^2/(2*b))/c
        end
    end
end

"""
    poisson_time(a[, u])

Obtaining waiting time for homogeneous Poisson Process
with rate of the form λ(t) = a, `a` ≥ 0, `u` uniform random variable
"""
function poisson_time(a::Number, u::Number)
    -log(u)/a
end

function poisson_time(rng, a)
    randexp(rng)/a
end
function poisson_time(a)
    randexp()/a
end

poisson_time((a, b)::Tuple, u=randn()) = poisson_time(a, b, u)


BigFloat(1.75)              # Accidentally OK, since 1.75 is exactly represented by Float64

BigFloat(2.1)               # First rounded to Float64, low precision

parse(BigFloat, "2.1")      # High precision

setprecision(512) do
    BigFloat(1) / BigFloat(3)
end

display(BigFloat(pi))
display(sin(BigFloat(pi)))

function halley_cuberoot(a, ε)
    x = 1    # Initial guess
    Δx = 1   # Need to initialize to get started
    while Δx ≥ ε
        xnew = x - 2(x^3 - a)*3x^2 / (2(3x^2)^2 - (x^3 - a)*(6x))
        Δx = abs(xnew - x)
        x = xnew
        println("Error = ", abs(x - cbrt(a)))
    end
    x
end

halley_cuberoot(BigFloat(2), 1e-60)


# ===========================================================
# Concrete overapproximation with a ball in the infinity norm
# ===========================================================

"""
    ballinf_approximation(S::LazySet)

Overapproximate a set by a tight ball in the infinity norm.

### Input

- `S` -- set

### Output

A tight ball in the infinity norm.

### Algorithm

The center and radius of the box are obtained by evaluating the support function
of the given convex set along the canonical directions.
"""
function ballinf_approximation(S::LazySet{N}) where {N}
    n = dim(S)
    c = Vector{N}(undef, n)
    r = zero(N)
    d = zeros(N, n)

    @inbounds for i in 1:n
        d[i] = one(N)
        htop = ρ(d, S)
        d[i] = -one(N)
        hbottom = -ρ(d, S)
        d[i] = zero(N)
        c[i] = (htop + hbottom) / 2
        rcur = (htop - hbottom) / 2
        if (rcur > r)
            r = rcur
        elseif rcur < 0
            # contradicting bounds => set is empty
            return EmptySet{N}(dim(S))
        end
    end
    return BallInf(c, r)
end

# ===============
# Specializations
# ===============

# empty set specialization
ballinf_approximation(∅::EmptySet) = ∅


export SoftPlus

"""
`SoftPlus([domainType=Float64::Type,] dim_in::Tuple)`

Creates the softplus non-linear operator with input dimensions `dim_in`.
```math
\\sigma(\\mathbf{x}) = \\log (1 + e^{x} )
```

"""
struct SoftPlus{T,N} <: NonLinearOperator
	dim::NTuple{N,Int}
end

function SoftPlus(DomainType::Type, DomainDim::NTuple{N,Int}) where {N}
	SoftPlus{DomainType,N}(DomainDim)
end

SoftPlus(DomainDim::NTuple{N,Int}) where {N} = SoftPlus{Float64,N}(DomainDim)

function mul!(y::AbstractArray{T,N}, L::SoftPlus{T,N}, x::AbstractArray{T,N}) where {T,N}
	y .= log.(1 .+exp.(x))
end

function mul!(y::AbstractArray,
              J::AdjointOperator{Jacobian{A,TT}},
              b::AbstractArray) where {T, N, A <: SoftPlus{T,N}, TT <: AbstractArray{T,N} }
    L = J.A
	y .= 1 ./(1 .+exp.(-L.x)).*b
end

fun_name(L::SoftPlus) = "σ"

size(L::SoftPlus) = (L.dim, L.dim)

domainType(L::SoftPlus{T,N}) where {T,N} = T
codomainType(L::SoftPlus{T,N}) where {T,N} = T


using BenchmarkTools
using Distributions
using GLM
using HypothesisTests
using LinearAlgebra
using Plots
using Random
using RDatasets
using SpecialFunctions
using Statistics

# Ridge regression

n = 10000
m = 1000

Random.seed!(666)
X = randn(n, m)

y = 10*X[:,1] + X[:,2] + randn(n)

# Exercise

ridge_reg(X, y, μ) = (X'*X + μ*I) \ (X'*y)

eigen_dec = eigen(X'*X)
Q = eigen_dec.vectors
Q_inv = Matrix(Q')
λ = eigen_dec.values

ridge_reg(X, y, μ, Q, Q_inv, λ) = Q * ((Diagonal(1 ./ (λ .+ μ)) * ( Q_inv * (X'*y))))

w1 = ridge_reg(X, y, 10)
w2 = ridge_reg(X, y, 10, Q, Q_inv, λ)

norm(w1 - w2)

# Comparison

@btime ridge_reg(X, y, 10);

@btime ridge_reg(X, y, 10, Q, Q_inv, λ);

# Dependence on mu

μs = range(0, 1000; length=50)

ws = hcat(ridge_reg.(Ref(X), Ref(y), μs, Ref(Q), Ref(Q_inv), Ref(λ))...)

plot(μs, abs.(ws');
    label="",
    yscale=:log10,
    xlabel="mu",
    ylabel="weights: log scale",
)

# LASSO

S(x, η) = max(x-η, 0) - max(-x-η, 0)

function lasso(X, y, μ, Q, Q_inv, λ;
        max_iter = 100,
        ρ = 1e3,
        w = zeros(size(X,2)),
        u = zeros(size(X,2)),
        z = zeros(size(X,2)),
    )

    for i in 1:max_iter
        w = Q * ( (Diagonal(1 ./ (λ .+ ρ)) * ( Q_inv * (X'*y + ρ*(z-u)))))
        z = S.(w + u, μ / ρ)
        u = u + w - z
    end
    return w, u, z
end

ws = zeros(size(X,2), length(μs))

for (i, μ) in enumerate(μs)
    global w, u, z
    w, u, z = i > 1 ? lasso(X, y, μ, Q, Q_inv, λ; w, u, z) : lasso(X, y, μ, Q, Q_inv, λ)
    ws[:,i] = w
end

plot(μs, abs.(ws');
    label="",
    yscale=:log10,
    xlabel="mu",
    ylabel="weights: log scale",
)

plot(0:0.1:10, gamma;
    xlabel="x",
    ylabel="gamma(x): log scale",
    label="",
    yscale=:log10,
)

# Exercise

volume_true(m, R) = π^(m/2) *R^2 / gamma(m/2 + 1)

plot(1:100, m -> volume_true.(m, 1);
    xlabel="dimension",
    ylabel="unit ball volume: log scale",
    label="",
    yscale=:log10,
)

# Exercise

function volume_monte_carlo(m::Int; n::Int=10000)
    X = 2*rand(m, n).-1
    X_norm_sq = sum(X.^2; dims=1)
    return 2^m*mean(X_norm_sq .<= 1)
end

# Volume of a ball

ms = 1:15
ns = Int64.([1e1; 1e3; 1e5])

Random.seed!(666)

plt = plot(ms, m -> volume_true(m, 1);
    xlabel="dimension",
    ylabel="unit ball volume",
    legend=:topleft,
    label="True",
    line=(4,:black),
)

for n in ns
    plot!(plt, ms, m -> volume_monte_carlo.(m; n=n); label="n = $n")
end

display(plt)

# Generating from the uniform distributions

rand(Uniform(-1, 1), 10, 5)

# Sampling from distributions

d1 = Normal()
d2 = Normal(1, 1)
d3 = Normal(0, 0.01)

f1(x) = pdf(d1, x)
f2(x) = pdf(d2, x)
f3(x) = pdf(d3, x)

function plot_histogram(xs, f; kwargs...)
    plt = histogram(xs;
        label="Sampled density",
        xlabel = "x",
        ylabel = "pdf(x)",
        nbins = 85,
        normalize = :pdf,
        opacity = 0.5,
        kwargs...
    )

    plot!(plt, range(minimum(xs), maximum(xs); length=100), f;
        label="True density",
        line=(4,:black),
    )

    return plt
end

plot_histogram(rand(d1, 1000000), f1)

# Exercise

function rejection_sampling(f, f_max, x_min, x_max; n=1000000)
    xs = x_min .+ (x_max - x_min)*rand(n)
    ps = f_max*rand(n)
    return xs[f.(xs) .>= ps]
end

# Rejection sampling

xlims = (-10, 10)

for (f, d) in zip((f1, f2, f3), (d1, d2, d3))
    Random.seed!(666)
    xs = rejection_sampling(f, f(d.μ), xlims...)

    pl = plot_histogram(xs, f)
    display(pl)
end

# Exercise

n = 1000
m = 9

Random.seed!(666)
xs = rand(m, n)

dist1 = [norm(x-y) for x in eachcol(xs), y in eachcol(xs)]
dist2 = [dist1[i,j] for i in 1:n for j in i+1:n]

extrema(dist2)

# Hypothesis testing

Random.seed!(666)

n = 1000
xs = randn(n)

# Exercise

t = mean(xs) / std(xs) * sqrt(n)

prob = cdf(TDist(n-1), t)
p = 2*min(prob, 1-prob)

# One-sample t-test

OneSampleTTest(xs)

# Linear models

wages = dataset("plm", "Snmesp")
wages.W = 2. .^ (wages.W)

X = Matrix(wages[:, [:N, :Y, :I, :K, :F]])
X = hcat(ones(size(X,1)), X)
y = wages[:, :W]

w0 = (X'*X) \ (X'*y)

model = lm(@formula(W ~ 1 + N + Y + I + K + F), wages)

# Exercise

norm(coef(model) - w0)

model_red = lm(@formula(W ~ 1 + N + Y + I + F), wages)

(r2(model), r2(model_red))

# Plot histograms

y_hat = predict(model)

plot_histogram(y_hat, x -> pdf(Normal(mean(y_hat), std(y_hat)), x))

plot_histogram(y_hat, x -> pdf(fit(Normal, y_hat), x))

test_normality = ExactOneSampleKSTest(y_hat, Normal(mean(y_hat), std(y_hat)))

# Generalized linear models

model = glm(@formula(W ~ 1 + N + Y + I + K + F), wages, InverseGaussian(), SqrtLink())

# Exercise

g_inv(z) = z^2

y_hat = g_inv.(X*coef(model))

scatter(y, y_hat;
    label="",
    xlabel="Label",
    ylabel="Prediction",
)


# Higham Test Matrices

"""
Hilbert Matrix
==============
The Hilbert matrix has `(i,j)` element `1/(i+j-1)`. It is
notorious for being ill conditioned. It is symmetric
positive definite and totally positive.

*Input options:*

+ [type,] dim: the dimension of the matrix.

+ [type,] row_dim, col_dim: the row and column dimensions.

*Groups:* ["inverse", "illcond", "symmetric", "posdef"]

*References:*

**M. D. Choi**, Tricks or treats with the Hilbert matrix,
Amer. Math. Monthly, 90 (1983), pp. 301-312.

**N. J. Higham**, Accuracy and Stability of Numerical Algorithms,
second edition, Society for Industrial and Applied Mathematics,
Philadelphia, PA, USA, 2002; sec. 28.1.
"""
function hilb(::Type{T}, m::Integer, n::Integer) where T
    # compute the Hilbert matrix
    H = zeros(T, m, n)
    for j = 1:n, i= 1:m
        @inbounds H[i,j] = one(T)/ (i + j - one(T))
    end
    return H
end
hilb(::Type{T}, n::Integer) where T = hilb(T, n, n)
hilb(args...) = hilb(Float64, args...)
hilb(::Type, args...) = throw(MethodError(hilb, Tuple(args)))

"""
Inverse of the Hilbert Matrix
=============================
*Input options:*

+ [type,] dim: the dimension of the matrix.

*Groups:* ["inverse", "illcond", "symmetric","posdef"]

*References:*

**M. D. Choi**, Tricks or treats with the Hilbert matrix,
    Amer. Math. Monthly, 90 (1983), pp. 301-312.

**N. J. Higham**, Accuracy and Stability of Numerical Algorithms, second
    edition, Society for Industrial and Applied Mathematics, Philadelphia, PA,
    USA, 2002; sec. 28.1.
"""
function invhilb(::Type{T}, n::Integer) where T
    # compute the inverse of Hilbert matrix
    # Type: data type
    # n: the dimension of the matrix
    Inv = zeros(T, n, n)
    for i = 1:n, j = 1:n
        Inv[i,j] = (-1)^(i+j)*(i+j-1)*binomial(n+i-1, n-j)*
        binomial(n+j-1,n-i)*binomial(i+j-2,i-1)^2
    end
    return Inv
end
invhilb(n::Integer) = invhilb(Float64, n)

"""
Hadamard Matrix
===============
The Hadamard matrix is a square matrix whose entries are
1 or -1. It was named after Jacques Hadamard. The rows of
a Hadamard matrix are orthogonal.

*Input options:*

+ [type,] dim: the dimension of the matrix, `dim` is a power of 2.

*Groups:* ["inverse", "orthogonal", "eigen"]

*References:*

**S. W. Golomb and L. D. Baumert**, The search for
Hadamard matrices, Amer. Math. Monthly, 70 (1963) pp. 12-17
"""
function hadamard(::Type{T}, n::Integer) where T
    #Compute the hadamard matrix
    if n < 1
        lgn = 0
    else
        lgn = round(Integer, log2(n))
    end
    2^lgn != n && throw(ArgumentError("n must be positive integer and a power of 2."))

    H = reshape(T[1], 1, 1)
    for i = 1:lgn
        H = [[H H]; [H -H]]
    end
    return H
end
hadamard(n::Integer) = hadamard(Float64, n)

"""
Cauchy Matrix
=============
Given two vectors `x` and `y`, the `(i,j)` entry of the Cauchy matrix is
`1/(x[i]+y[j])`.

*Input options*:

 + [type,] x, y: two vectors.

 + [type,] x: a vector. `y` defaults to `x`.

 + [type,] dim: the dimension of the matrix. `x` and `y` default to
    `[1:dim;]`.

*Groups:* ["inverse", "illcond", "symmetric", "posdef"]

*References:*

**N. J. Higham**, Accuracy and Stability of Numerical Algorithms,
second edition, Society for Industrial and Applied Mathematics, Philadelphia, PA, USA,
2002; sec. 28.1
"""
function cauchy(::Type{T}, x::Vector, y::Vector) where T
    # Compute the cauchy matrix
    m = size(x,1)
    n = size(y,1)
    C = zeros(T, m, n)
    [C[i,j]= one(T)/(x[i] + y[j]) for i = 1:m, j = 1:n]
    return C
end

cauchy(::Type{T}, x::Vector) where T = cauchy(T, x, x)
cauchy(::Type{T}, k::Integer) where T = cauchy(T, [1:k;])
cauchy(arg...) = cauchy(Float64, arg...)
cauchy(::Type, args...) = throw(MethodError(cauchy, Tuple(args)))

"""
Circulant Matrix
================
A circulant matrix has the property that each row is obtained
by cyclically permuting the entries of the previous row one
step forward.

*Input options:*

+ [type,] vec, col_dim: a vector and the column dimension.

+ [type,] vec: a vector.

+ [type,] dim: the dimension of the matrix.

*Groups:* ["symmetric", "posdef", "eigen"]

*References:*

**P. J. Davis**, Circulant Matrices, John Wiley, 1977.
"""
function circul(::Type{T}, v::Vector, w::Vector) where T
    # Compute the circul matrix
    # v: the first row of the matrix
    # w: the length of the vector is the dimension of the column
    m = length(v)
    n = length(w)
    C = zeros(T, m, n)
    [C[i,j] = v[1 + mod(j - i, n)] for i = 1:m, j = 1:n]
    return C
end
circul(::Type{T}, v::Vector, n::Integer) where T = circul(T, v, T[1:n;])
circul(::Type{T}, v::Vector) where T = circul(T, v, v)
circul(::Type{T}, k::Integer) where T = circul(T, [1:k;])
circul(arg...) = circul(Float64, arg...)

"""
Dingdong Matrix
===============
The Dingdong matrix is a symmetric Hankel matrix invented
by DR. F. N. Ris of IBM, Thomas J Watson Research Centre.
The eigenvalues cluster around `π/2` and `-π/2`.

*Input options:*

+ [type,] dim: the dimension of the matrix.

*Groups:* ["symmetric", "eigen"]

*References:*

**J. C. Nash**, Compact Numerical Methods for
Computers: Linear Algebra and Function Minimisation,
second edition, Adam Hilger, Bristol, 1990 (Appendix 1).
"""
function dingdong(::Type{T}, n::Integer) where T
    # Compute the dingdong matrix
    # type: data type
    # n: the dimension of the matrix
    D = zeros(T, n, n)
    [D[i,j] = one(T)/(2*(n-i-j+1.5)) for i = 1:n, j= 1:n]
    return D
end
dingdong(args...) = dingdong(Float64, args...)
dingdong(::Type, args...) = throw(MethodError(dingdong, Tuple(args)))

"""
Frank Matrix
============
The Frank matrix is an upper Hessenberg matrix with
determinant 1. The eigenvalues are real, positive and
very ill conditioned.

*Input options:*

+ [type,] dim, k: `dim` is the dimension of the matrix, `k = 0 or 1`.
    If `k = 1` the matrix reflect about the anti-diagonal.

+ [type,] dim: the dimension of the matrix.

*Groups:* ["illcond", "eigen"]

*References:*

**W. L. Frank**, Computing eigenvalues of complex matrices
    by determinant evaluation and by methods of Danilewski and Wielandt,
    J. Soc. Indust. Appl. Math., 6 (1958), pp. 378-392 (see pp. 385, 388).
"""
function frank(::Type{T}, n::Integer, k::Integer) where T
    # Compute the frank matrix
    # type: data type
    # n: the dimension of the matrix
    # k = 0 or 1: k = 1 reflect about the anti-diagonal
    p = T[n:-1:1;];
    F = triu(ones(T, n, 1)*p') + diagm(-1 => p[2:n]);
    k == 0 ?  F :
    k == 1 ?  F[p,p]' : error("k = 0 or 1, but get ", k)
end
frank(::Type{T}, n::Integer) where T = frank(T, n, 0)
frank(args...) = frank(Float64, args...)
frank(::Type, args...) = throw(MethodError(frank, Tuple(args)))

#
# Jordan block
#
function jordbloc( n::Integer, lambda::T) where T
    # Generate a n-by-n jordan block with eigenvalue
    # lambda
    J = lambda*Matrix{T}(I, n, n) + diagm(1 => ones(T, n-1, 1)[:,1])
    return J
end

"""
Forsythe Matrix
===============
The Forsythe matrix is a n-by-n perturbed Jordan block.
This generator is adapted from N. J. Higham's Test Matrix Toolbox.

*Input options:*

+ [type,] dim, alpha, lambda: `dim` is the dimension of the matrix.
    `alpha` and `lambda` are scalars.

+ [type,] dim: `alpha = sqrt(eps(type))` and `lambda = 0`.

*Groups:* ["inverse", "illcond", "eigen"]
"""
function forsythe(::Type{T}, n::Integer , alpha, lambda) where T
    # Generate the forsythe matrix
    alpha = convert(T, alpha)
    lambda = convert(T, lambda)
    F = jordbloc(n, lambda);
    F[n,1] = alpha;
    return F
end
forsythe(::Type{T}, n::Integer) where T = forsythe(T, n, sqrt(eps(T)), zero(T))
forsythe(args...) = forsythe(Float64, args...)
forsythe(::Type, args...) = throw(MethodError(forsythe, Tuple(args)))

function oddmagic(::Type{T}, n::Integer) where T
    # compute the magic square of odd orders
    A = zeros(T, n, n)
    i = 1
    j = div(n+1, 2)
    for k = 1:n^2
        is = i
        js = j
        A[i,j] = k
        i = n - rem(n+1-i,n)
        j = rem(j,n) + 1
        if A[i,j] != 0
            i = rem(is,n) + 1
            j = js
        end
    end
    return A
end

"""
Magic Square Matrix
===================
The magic matrix is a matrix with integer entries such that
    the row elements, column elements, diagonal elements and
    anti-diagonal elements all add up to the same number.

*Input options:*

+ [type,] dim: the dimension of the matrix.

*Groups:* ["inverse"]
"""
function magic(::Type{T}, n::Integer) where T
    # Compute a magic square of order n
    # Learnt from Cleve Moler, Experiments with MATLAB, 2011
    if mod(n, 2) == 1
        # n is odd
        M = oddmagic(T, n)
    elseif mod(n, 4) == 0
        # n is doubly even
        a = floor.(Integer, mod.([1:n;], 4)/2)
        B = broadcast(==, a', a)
        M = broadcast(+, T[1:n:n^2;]',T[0:n-1;])
        for i = 1:n, j = 1:n
            B[i,j] == 1 ? M[i,j] = n^2 + one(T) - M[i,j] : M[i,j]
        end
    else
        # n is singly even
        p = div(n,2)
        M = oddmagic(T, p)
        M = [M M.+2*p^2; M.+3*p^2 M.+p^2]
        if n == 2
            return M
        end
        i = [1:p;]
        k = div(n-2, 4)
        j = [[1:k;]; [(n-k+2) : n;]]
        M[[i;i.+p],j] = M[[i.+p;i],j]
        i = k+1
        j = [1, i]
        M[[i;i+p],j] = M[[i+p;i],j]
    end
    return M
end
magic(n::Integer) = magic(Float64, n)

"""
Grcar Matrix
============
The Grcar matrix is a Toeplitz matrix with sensitive
eigenvalues.

*Input options:*

+ [type,] dim, k: `dim` is the dimension of the matrix and
    `k` is the number of superdiagonals.

+ [type,] dim: the dimension of the matrix.

*Groups:* ["eigen"]

*References:*

**J. F. Grcar**, Operator coefficient methods
    for linear equations, Report SAND89-8691, Sandia National
    Laboratories, Albuquerque, New Mexico, 1989 (Appendix 2).
"""
function grcar(::Type{T}, n::Integer, k::Integer = 3) where T
    # Compute grcar matrix
    G = tril(triu(ones(T, n,n)), min(k, n-1)) - diagm(-1 => ones(T, n-1))
    return G
end
grcar(args...) = grcar(Float64, args...)
grcar(::Type, args...) = throw(MethodError(grcar, Tuple(args)))

"""
Triw Matrix
===========
Upper triangular matrices discussed by Wilkinson and others.

*Input options:*

+ [type,] row_dim, col_dim, α, k: `row_dim` and `col_dim`
        are row and column dimension of the matrix. `α` is a
        scalar representing the entries on the superdiagonals.
        `k` is the number of superdiagonals.

+ [type,] dim: the dimension of the matrix.

*Groups:* ["inverse", "illcond"]

*References:*

**G. H. Golub and J. H. Wilkinson**, Ill-conditioned
eigensystems and the computation of the Jordan canonical form,
SIAM Review, 18(4), 1976, pp. 578-6
"""
function triw(::Type{T}, m::Integer, n::Integer, alpha, k::Integer) where T
    alpha = convert(T, alpha)
    A = tril(Matrix{T}(I, m, n) + alpha * triu(ones(T, m,n), 1), min(k, n-1))
    return A
end
triw(::Type{T}, n::Integer) where T = triw(T, n, n,  -1, n-1)
triw(args...) = triw(Float64, args...)
triw(::Type, args...) = throw(MethodError(triw, Tuple(args)))

"""
Moler Matrix
============
The Moler matrix is a symmetric positive definite matrix.
It has one small eigenvalue.

*Input options:*

+ [type,] dim, alpha: `dim` is the dimension of the matrix,
        `alpha` is a scalar.

+ [type,] dim: `alpha = -1`.

*Groups:* ["inverse", "illcond", "symmetric", "posdef"]

*References:*

**J.C. Nash**, Compact Numerical Methods for Computers:
    Linear Algebra and Function Minimisation, second edition,
    Adam Hilger, Bristol, 1990 (Appendix 1).
"""
function moler(::Type{T}, n::Integer, alpha = -1.) where T
    alpha = convert(T, alpha)
    M = triw(T, n, n, alpha, n - 1)' * triw(T, n, n, alpha, n -1 )
    return M
end
moler(args...) = moler(Float64, args...)
moler(::Type, args...) = throw(MethodError(moler, Tuple(args)))

"""
Pascal Matrix
=============
The Pascal matrix’s anti-diagonals form the Pascal’s triangle.

*Input options:*

+ [type,] dim: the dimension of the matrix.

*Groups:* ["inverse", "illcond", "symmetric", "posdef", "eigen"]

*References:*

**R. Brawer and M. Pirovino**, The linear algebra of
    the Pascal matrix, Linear Algebra and Appl., 174 (1992),
    pp. 13-23 (this paper gives a factorization of L = PASCAL(N,1)
               and a formula for the elements of L^k).

**N. J. Higham**, Accuracy and Stability of Numerical Algorithms,
second edition, Society for Industrial and Applied Mathematics, Philadelphia, PA,
USA, 2002; sec. 28.4.
"""
function pascal(::Type{T}, n::Integer) where T
    P = zeros(T, n,n)
    for j = 1:n
        for i = 1:n
            try P[i,j] = binomial(i+j-2, j-1)
            catch
                P[i,j] = binomial(BigInt(i+j-2), j-1)
            end
        end
    end
    return P
end
pascal(n::Integer) = pascal(Float64, n)

"""
Kahan Matrix
============
The Kahan matrix is an upper trapezoidal matrix, i.e., the
`(i,j)` element is equal to `0` if `i > j`. The useful range of
    `θ` is `0 < θ < π`. The diagonal is perturbed by
    `pert*eps()*diagm([n:-1:1;])`.

*Input options:*

+ [type,] rowdim, coldim, θ, pert: `rowdim` and `coldim` are the row and column
    dimensions of the matrix. `θ` and `pert` are scalars.

+ [type,] dim, θ, pert: `dim` is the dimension of the matrix.

+ [type,] dim: `θ = 1.2`, `pert = 25`.

*Groups:* ["inverse", "illcond"]

*References:*

**W. Kahan**, Numerical linear algebra, Canadian Math.
    Bulletin, 9 (1966), pp. 757-801.
"""
function kahan(::Type{T}, m::Integer, n::Integer, theta, pert) where T
    theta = convert(T, theta)
    pert = convert(T, pert)
    s = sin(theta)
    c = cos(theta)
    dim = min(m,n)
    A = zeros(T, m, n)
    for i = 1:m, j = 1:n
        i > dim ? A[i,j] = zero(T) :
        i > j   ? A[i,j] = zero(T) :
        i==j    ? A[i,j] = s^(i-1)+pert*eps(T)*(m-i+1) : A[i,j] = -c*s^(i-1)
    end
    return A
end
kahan(::Type{T}, n::Integer, theta, pert) where T = kahan(T, n, n, theta, pert)
kahan(::Type{T}, n::Integer) where T = kahan(T, n, n, 1.2, 25.)
kahan(args...) = kahan(Float64, args...)
kahan(::Type, args...) = throw(MethodError(kahan, Tuple(args)))

"""
Pei Matrix
==========
The Pei matrix is a symmetric matrix with known inversion.

*Input options:*

+ [type,] dim, α: `dim` is the dimension of the matrix.
    `α` is a scalar.

+ [type,] dim: the dimension of the matrix.

*Groups:* ["inverse", "illcond", "symmetric", "posdef"]

*References:*

**M. L. Pei**, A test matrix for inversion procedures,
    Comm. ACM, 5 (1962), p. 508.
"""
function pei(::Type{T}, n::Integer, alpha = 1) where T
    alpha = convert(T, alpha)
    return alpha*Matrix{T}(I, n, n) + ones(T, n, n)
end
pei(args...) = pei(Float64, args...)
pei(::Type, args...) = throw(MethodError(pei, Tuple(args)))

"""
Vandermonde Matrix
==================
The inverse and determinat are known explicity.

*Input options:*

+ [type,] vec, col_dim: `vec` is a vector, `col_dim` is the number of columns.

+ [type,] vec: `col_dim = length(vec)`

+ [type,] dim: `vec = [1:dim;]`

*Groups:* ["inverse", "illcond"]

*References:*

**N. J. Higham**, Stability analysis of algorithms
    for solving confluent Vandermonde-like systems, SIAM J.
        Matrix Anal. Appl., 11 (1990), pp. 23-41.
"""
function vand(::Type{T}, p::Vector, n::Integer) where T
    # n: number of rows
    # p: a vector
    m = length(p)
    V = Array{T,2}(undef, m, n)
    for j = 1:m
        @inbounds V[j, 1] = 1
    end
    for i = 2:n
        for j = 1:m
            @inbounds V[j,i] = p[j] * V[j,i-1]
        end
    end
    return V
end
vand(::Type{T}, n::Integer) where T = vand(T, [1:n;], n)
vand(::Type{T}, p::Vector) where T = vand(T, p, length(p))
vand(args...) = vand(Float64, args...)
vand(::Type, args...) = throw(MethodError(vand, Tuple(args)))

"""
Involutory Matrix
=================
An involutory matrix is a matrix that is its own inverse.

*Input options:*

+ [type,] dim: `dim` is the dimension of the matrix.

*Groups:* ["inverse", "illcond", "eigen"]

*References:*

**A. S. Householder and J. A. Carpenter**, The
        singular values of involutory and of idempotent matrices,
        Numer. Math. 5 (1963), pp. 234-237.
"""
function invol(::Type{T}, n::Integer) where T
    A = hilb(T, n)
    d = -n
    A[:,1] = d*A[:, 1]
    for i = 1:n-1
        d = -(n+i)*(n-i)*d/(i*i)
        A[i+1,:] = d*A[i+1,:]
    end
    return A
end
invol(n::Integer) = invol(Float64, n)

"""
Chebyshev Spectral Differentiation Matrix
=========================================
If `k = 0`,the generated matrix is nilpotent and a vector with
        all one entries is a null vector. If `k = 1`, the generated
        matrix is nonsingular and well-conditioned. Its eigenvalues
        have negative real parts.

*Input options:*

+ [type,] dim, k: `dim` is the dimension of the matrix and
        `k = 0 or 1`.

+ [type,] dim: `k=0`.

*Groups:* ["eigen"]

*References:*

**L. N. Trefethen and M. R. Trummer**, An instability
        phenomenon in spectral methods, SIAM J. Numer. Anal., 24 (1987), pp. 1008-1023.
"""
function chebspec(::Type{T}, n::Integer, k::Integer = 0) where T
    # k = 0 or 1
    if k == 1
        n = n + 1
    end
    c = ones(T, n)
    c[1] = 2
    c[2:n-1] .= 1
    c[n] = 2
    x = ones(T, n)

    A = zeros(T, n, n)
    # Compute the chebyshev points
    for i = 1:n
        x[i] = cos(pi * (i - 1) / (n - 1))
    end

    for j = 1:n, i = 1:n
        i != j ? A[i,j] = (-1)^(i+j) * c[i] / (c[j] * (x[i] - x[j])) :
        i == 1 ? A[i,i] = (2 * (n -1)^2 + 1) / 6 :
        i == n ? A[i,i] = - (2 * (n-1)^2 + 1) / 6 :
        A[i,i] = - 0.5 * x[i] / (1 - x[i]^2)
    end
    if k == 1
        A = A[2:n, 2:n]
    end
    return A
end
chebspec(args...) = chebspec(Float64, args...)
chebspec(::Type, args...) = throw(MethodError(chebspec, Tuple(args)))

"""
Lotkin Matrix
=============
The Lotkin matrix is the Hilbert matrix with its first row
        altered to all ones. It is unsymmetric, illcond and
        has many negative eigenvalues of small magnitude.

*Input options:*

+ [type,] dim: `dim` is the dimension of the matrix.

*Groups:* ["inverse", "illcond", "eigen"]

*References:*

**M. Lotkin**, A set of test matrices, MTAC, 9 (1955), pp. 153-161.
"""
function lotkin(::Type{T}, n::Integer) where T
    A = hilb(T, n)
    A[1,:] = ones(T,n)'
    return A
end
lotkin(n::Integer) = lotkin(Float64, n)

"""
Clement Matrix
==============
The Clement matrix is a tridiagonal matrix with zero
        diagonal entries. If k = 1, the matrix is symmetric.

*Input options:*

+ [type,] dim, k: `dim` is the dimension of the matrix.
        If `k = 0`, the matrix is of type `Tridiagonal`.
        If `k = 1`, the matrix is of type `SymTridiagonal`.

+ [type,] dim: `k = 0`.

*Groups:* ["inverse", "symmetric", "eigen"]

*References:*

**P. A. Clement**, A class of triple-diagonal
        matrices for test purposes, SIAM Review, 1 (1959), pp. 50-52.
"""
function clement(::Type{T}, n::Integer, k::Integer = 0) where T
    # construct Tridiagonal matrix
    # n is the dimension of the matrix
    # k = 0 or 1
    if n == 1 # handle the 1-d case.
        return zeros(T, 1, 1)
    end
    n = n -1
    x = T[n:-1:1;]
    z = T[1:n;]
    if k == 0
        A = Tridiagonal(x,zeros(T,n+1),z)
    else
        y = sqrt.(x.*z)
        A = SymTridiagonal(zeros(T,n+1), y)
    end
    return A
end
clement(args...) = clement(Float64, args...)
clement(::Type, args...) = throw(MethodError(clement, Tuple(args)))

"""
Fiedler Matrix
==============
The Fiedler matrix is symmetric matrix with a dominant
      positive eigenvalue and all the other eigenvalues are negative.

*Input options:*

+ [type,] vec: a vector.

+ [type,] dim: `dim` is the dimension of the matrix. `vec=[1:dim;]`.

*Groups: *["inverse", "symmetric", "eigen"]

*References:*

**G. Szego**, Solution to problem 3705, Amer. Math.
            Monthly, 43 (1936), pp. 246-259.

**J. Todd**, Basic Numerical Mathematics, Vol. 2: Numerical Algebra,
            Birkhauser, Basel, and Academic Press, New York, 1977, p. 159.
"""
function fiedler(::Type{T}, v::Vector) where T
    n = length(v)
    v = transpose(v[:])
    A = ones(T, n) * v
    A = abs.(A - transpose(A)) # nonconjugate transpose
end
fiedler(::Type{T}, n::Integer) where T = fiedler(T, [1:n;])
fiedler(args...) = fiedler(Float64, args...)
fiedler(::Type, args...) = throw(MethodError(fiedler, Tuple(args)))

"""
MIN[I,J] Matrix
===============
A matrix with `(i,j)` entry `min(i,j)`. It is a symmetric positive
     definite matrix. The eigenvalues and eigenvectors are known
     explicitly. Its inverse is tridiagonal.

*Input options:*

+ [type,] dim: the dimension of the matrix.

*Groups:* ["inverse", "symmetric", "posdef", "eigen"]

*References:*

**J. Fortiana and C. M. Cuadras**, A family of matrices,
            the discretized Brownian bridge, and distance-based regression,
            Linear Algebra Appl., 264 (1997), 173-188.  (For the eigensystem of A.)
"""
function minij(::Type{T}, n::Integer) where T
    A = zeros(T, n, n)
    [A[i,j] = min(i,j) for i = 1:n, j = 1:n]
    return A
end
minij(n::Integer) = minij(Float64, n)

"""
Binomial Matrix
===============
The matrix is a multiple of an involutory matrix.

*Input options:*

+ [type,] dim: the dimension of the matrix.
"""
function binomialm(::Type{T}, n::Integer) where T
    # Mulitiple of involutory matrix
    L = Array{T,2}(undef, n, n)
    D = Diagonal((-2).^[0:n-1;])
    [L[i,j] = binomial(i-1, j-1) for i = 1:n, j = 1:n]
    U = L[n:-1:1, n:-1:1]
    return L*D*U
end
binomialm(n::Integer) = binomialm(Float64, n)

"""
Tridiagonal Matrix
==================
Construct a tridigonal matrix of type `Tridiagonal`.

*Input options:*

+ [type,] v1, v2, v3: `v1` and `v3` are vectors of subdiagonal
            and superdiagonal elements, respectively, and `v2` is a vector
            of diagonal elements.

+ [type,] dim, x, y, z: `dim` is the dimension of the matrix,
            `x`, `y`, `z` are scalars. `x` and `z` are the subdiagonal and
            superdiagonal elements, respectively, and `y` is the diagonal
            elements.

+ [type,] dim: `x = -1, y = 2, z = -1`. This matrix is also
            known as the second difference matrix.

*Groups:* ["inverse", "illcond", "posdef", "eigen"]

*References:*

**J. Todd**, Basic Numerical Mathematics, Vol. 2:
            Numerical Algebra, Birkhauser, Basel, and Academic Press,
            New York, 1977, p. 155.
"""
function tridiag(::Type{T}, x::AbstractVector, y::AbstractVector,
                    z::AbstractVector) where T
    x = map((i)-> convert(T, i), x)
    y = map((i)-> convert(T, i), y)
    z = map((i)-> convert(T, i), z)
    return Tridiagonal(x,y,z)
end
# Toeplitz tridiagonal matrix
tridiag(::Type{T}, n::Integer, x::Integer, y::Integer, z::Integer) where T =
n == 1 ? y*ones(T,1,1) :
         tridiag(T, x*ones(T, n-1), y*ones(T, n), z*ones(T, n-1))
tridiag(::Type{T}, n::Integer) where T = tridiag(T, n, -1, 2, -1)
tridiag(args...) = tridiag(Float64, args...)
tridiag(::Type, args...) = throw(MethodError(tridiag, Tuple(args)))

"""
Lehmer Matrix
=============
The Lehmer matrix is a symmetric positive definite matrix.
            It is totally nonnegative. The inverse is tridiagonal and
            explicitly known

*Input options:*

+ [type,] dim: the dimension of the matrix.

*Groups:* ["inverse", "symmetric", "posdef"]

*References:*

**M. Newman and J. Todd**, The evaluation of
            matrix inversion programs, J. Soc. Indust. Appl. Math.,
            6 (1958), pp. 466-476.
            Solutions to problem E710 (proposed by D.H. Lehmer): The inverse
            of a matrix, Amer. Math. Monthly, 53 (1946), pp. 534-535.
"""
function lehmer(::Type{T}, n::Integer) where T
    A = Array{T,2}(undef, n, n)
    [A[i,j] = min(i,j) / max(i,j) for i = 1:n, j = 1:n]
    return A
end
lehmer(n::Integer) = lehmer(Float64, n)
#=
function lehmer(::Type{T}, n::Integer) where T
    A = Array{T,2}(n, n)
    [A[i,j] = min(i,j) / max(i,j) for i = 1:n, j = 1:n]
    return A
end
lehmer(n::Integer) = lehmer(Float64, n)
=#
"""
Parter Matrix
=============
The Parter matrix is a Toeplitz and Cauchy matrix
            with singular values near `π`.

*Input options:*

+ [type,] dim: the dimension of the matrix.

*Groups:* ["eigen"]

*References:*

The MathWorks Newsletter, Volume 1, Issue 1,
            March 1986, page 2. S. V. Parter, On the distribution of the
            singular values of Toeplitz matrices, Linear Algebra and
            Appl., 80 (1986), pp. 115-130.
"""
function parter(::Type{T}, n::Integer) where T
    A = Array{T,2}(undef, n, n)
    [A[i,j] = one(T) / (i - j + 0.5) for i = 1:n, j = 1:n]
    return A
end
parter(n::Integer) = parter(Float64, n)

"""
Chow Matrix
===========
The Chow matrix is a singular Toeplitz lower Hessenberg matrix.

*Input options:*

+ [type,] dim, alpha, delta: `dim` is dimension of the matrix.
            `alpha`, `delta` are scalars such that `A[i,i] = alpha + delta` and
            `A[i,j] = alpha^(i + 1 -j)` for `j + 1 <= i`.

+ [type,] dim: `alpha = 1`, `delta = 0`.

*Groups:* ["eigen"]

*References:*

**T. S. Chow**, A class of Hessenberg matrices with known
                eigenvalues and inverses, SIAM Review, 11 (1969), pp. 391-395.
"""
function chow(::Type{T}, n::Integer, alpha, delta) where T
    A = zeros(T, n, n)
    alpha = convert(T, alpha)
    delta = convert(T, delta)
    for i = 1:n, j = 1:n
        if i == j - 1
            A[i,j] = one(T)
        elseif i == j
            A[i,j] = alpha + delta
        elseif j + 1 <= i
            A[i,j] = alpha^(i + 1 - j)
        end
    end
    return A
end
chow(::Type{T}, n::Integer) where T = chow(T, n, 1, 0)
chow(args...) = chow(Float64, args...)
chow(::Type, args...) = throw(MethodError(chow, Tuple(args)))

#
# newsign: newsign(0) = 1
#
function newsign(x)
    x == 0 ? y = 1 : y = sign(x)
    return y
end

"""
Random Correlation Matrix
=========================
A random correlation matrix is a symmetric positive
     semidefinite matrix with 1s on the diagonal.

*Input options:*

+ [type,] dim: the dimension of the matrix.

*Groups:* ["symmetric", 'pos-semidef', "random"]
"""
function randcorr(::Type{T}, n::Integer) where T
    x = rand(T,n) # x is the vector of random eigenvalues from a uniform distribution.
    x = n * x / sum(x) # x has nonnegtive elements.
    A = diagm(0 => x)
    F = qr(randn(n,n));
    Q = F.Q*diagm(0 => sign.(diag(F.R))) # form a random orthogonal matrix.
    copyto!(A, Q*A*Q')

    a = diag(A)
    l = findall(a .< 1)
    g = findall(a .> 1)

    # Apply Given rotation to set A[i,i] = 1
    while length(l) > 0 && length(g) > 0
        k =  ceil(Integer, rand()*length(l))
        h =  ceil(Integer, rand()*length(g))
        i = l[k]
        j = g[h]
        if i > j
            i,j = j,i
        end
        alpha = sqrt(A[i,j]^2 - (a[i] - 1)*(a[j] - 1))
        # take sign to avoid cancellation.
        t = (A[i,j] + newsign(A[i,j]) * alpha) / (a[j] - 1)
        c = 1/ sqrt(1 + t^2)
        s = t*c

        A[:, [i,j]] = A[:, [i,j]] * [c s; -s c]
        A[[i,j], :] = [c -s; s c] * A[[i,j], :]

        A[i,i] = 1
        a = diag(A)
        l = findall(a.<1)
        g = findall(a.>1)
    end
    [A[i,i] = 1 for i = 1:n]
    return (A + A')/2

end
randcorr(args...) = randcorr(Float64, args...)
randcorr(::Type, args...) = throw(MethodError(randcorr, Tuple(args)))

"""
Poisson Matrix
==============
A block tridiagonal matrix from Poisson’s equation.
     This matrix is sparse, symmetric positive definite and
     has known eigenvalues. The result is of type `SparseMatrixCSC`.

*Input options:*

+ [type,] dim: the dimension of the matirx is `dim^2`.

*Groups:* ["inverse", "symmetric", "posdef", "eigen", "sparse"]

*References:*

**G. H. Golub and C. F. Van Loan**, Matrix Computations,
          second edition, Johns Hopkins University Press, Baltimore,
          Maryland, 1989 (Section 4.5.4).
"""
function poisson(::Type{T}, n::Integer) where T
    S = Array(tridiag(T, n))
    A = sparse(T(1)I, n, n)
    return kron(A,S) + kron(S,A)
end
poisson(n::Integer) = poisson(Float64, n)

"""
Toeplitz Matrix
===============
A Toeplitz matrix is a matrix in which each descending
       diagonal from left to right is constant.

*Input options:*

+ [type,] vc, vr: `vc` and `vr` are the first column and row of the matrix.

+ [type,] v: symmatric case, i.e., `vc = vr = v`.

+ [type,] dim: `dim` is the dimension of the matrix. `v = [1:dim;]` is the first
                row and column vector.
"""
function toeplitz(::Type{T}, vc::Vector, vr::Vector) where T
    n = length(vc)
    length(vr) == n || throw(DimensionMismatch(""))
    vc[1] == vr[1] || error("The first element of the vectors must be the same.")
    A = Array{T,2}(undef, n, n)
    [i>=j ? A[i,j] = vc[i-j+1] : A[i,j] = vr[j-i+1] for i=1:n, j=1:n]
    A
end
toeplitz(::Type{T}, v::Vector) where T = toeplitz(T, v, v)
toeplitz(::Type{T}, n::Integer) where T = toeplitz(T, [1:n;])
toeplitz(args...) = toeplitz(Float64, args...)
toeplitz(::Type, args...) = throw(MethodError(toeplitz, Tuple(args)))

"""
Hankel Matrix
=============
A Hankel matrix is a matrix that is symmetric and constant
                across the anti-diagonals.

*Input options:*

+ [type,] vc, vr: `vc` and `vc` are the first column and last row of the
       matrix. If the last element of `vc` differs from the first element
                of `vr`, the last element of `rc` prevails.

+ [type,] v: `vc = vr = v`.

+ [type,] dim: `dim` is the dimension of the matrix. `v = [1:dim;]`.
"""
function hankel(::Type{T}, vc::Vector, vr::Vector) where T
    p = [vc; vr[2:end]]
    m = length(vc)
    n = length(vr)
    H = Array{T,2}(undef, m, n)
    [H[i,j] = p[i+j-1] for i=1:m, j=1:n]
    H
end
hankel(::Type{T}, v::Vector) where T = hankel(T, v, v)
hankel(::Type{T}, n::Integer) where T = hankel(T, [1:n;])
hankel(args...) = hankel(Float64, args...)
hankel(::Type, args...) = throw(MethodError(hankel, Tuple(args)))

"""
Prolate Matrix
==============
A prolate matrix is a symmetirc, ill-conditioned Toeplitz matrix.

*Input options:*

+ [type,] dim, w: `dim` is the dimension of the matrix. `w` is a real scalar.

+ [type,] dim: the case when `w = 0.25`.

*References:*

**J. M. Varah**. The Prolate Matrix. Linear Algebra and Appl.
             187:267--278, 1993.
"""
function prolate(::Type{T}, n::Integer, w::Real) where T
    v = Array{T,1}(undef, n)
    v[1] = 2*w
    [v[i] = sin(2*pi*w*i)/pi*i for i = 2:n]
    return toeplitz(T, v)
end
prolate(::Type{T}, n::Integer) where T = prolate(T, n, 0.25)
prolate(args...) = prolate(Float64, args...)
prolate(::Type, args...) = throw(MethodError(prolate, Tuple(args)))

"""
Neumann Matrix
==============
A singular matrix from the discrete Neumann problem.
       The matrix is sparse and the null space is formed by a vector of ones

*Input options:*

+ [type,] dim: the dimension of the matrix is `dim^2`.

*Groups:* ["eigen", "sparse"]

*References:*

**R. J. Plemmons**, Regular splittings and the
          discrete Neumann problem, Numer. Math., 25 (1976), pp. 153-161.
"""
function neumann(::Type{T}, n::Integer) where T
    if n == 1
        return 4 * ones(T, 1,1) #handle 1-d case.
    end
    S = Matrix(tridiag(T, n))
    S[1,2] = -2
    S[n, n-1] = -2
    A = sparse(T(1)I, n, n)
    return kron(S,A) + kron(A,S)
end
neumann(n::Integer) = neumann(Float64, n)

#
# Sylvester's orthogonal matrix
# See Rosser matrix References 2.
#
# for a = d = 2, b = c = 1, P_block' * P_block = 10 * Identity
#
P_block(::Type{T}, a, b, c, d) where T =
    reshape(T[a, b, c, d, b, -a, -d, c, c, d, -a, -b, d, -c, b, -a], 4,4)

"""
Rosser Matrix
=============
The Rosser matrix’s eigenvalues are very close together
        so it is a challenging matrix for many eigenvalue algorithms.

*Input options:*

+ [type,] dim, a, b: `dim` is the dimension of the matrix.
            `dim` must be a power of 2.
            `a` and `b` are scalars. For `dim = 8, a = 2` and `b = 1`, the generated
            matrix is the test matrix used by Rosser.

+ [type,] dim: `a = rand(1:5), b = rand(1:5)`.

*Groups:* ["eigen", "illcond", "random"]

*References:*

**J. B. Rosser, C. Lanczos, M. R. Hestenes, W. Karush**,
            Separation of close eigenvalues of a real symmetric matrix,
            Journal of Research of the National Bureau of Standards, v(47)
            (1951)
"""
function rosser(::Type{T}, n::Integer, a, b) where T
    if n < 1
        lgn = 0
    else
        lgn = round(Integer, log2(n))
    end
    2^lgn != n && throw(ArgumentError("n must be positive integer and a power of 2."))

    if n == 1 # handle 1-d case
        return 611 * ones(T, 1, 1)
    end

    if n == 2
        #eigenvalues are 500, 510
        B = T[101 1; 1 101]
        P = T[2 1;1 -2]
        A = P'*B*P
    elseif n == 4
        # eigenvalues are 0.1, 1019.9, 1020, 1020 for a = 2 and b = 1
        B = zeros(T, n, n)
        B[1,1], B[1,4], B[4,1], B[4,4] = 101, 1, 1, 101;
        B[2,2], B[2,3], B[3,2], B[3,3] = 1, 10, 10, 101;
        P = P_block(T, a, b, b, a)
        A = P' * B * P
    elseif n == 8
        # eigenvalues are 1020, 1020, 1000, 1000, 0.098, 0, -1020
        B = zeros(T, n, n)
        B[1,1], B[6,1], B[2,2], B[8,2] = 102, 1, 101, 1;
        B[3,3], B[7,3] = 98, 14;
        B[4,4], B[5,4], B[4,5], B[5,5] = 1, 10, 10, 101;
        B[1,6], B[6,6], B[3,7],B[7,7], B[2,8], B[8,8] = 1, -102, 14, 2, 1, 101;
        P = [P_block(T, a, b, b, a)' zeros(T, 4,4); zeros(T, 4,4) P_block(T, b, -b, -a, a)]
        A = P' * B * P
    else
        lgn = lgn - 2
        halfn = round(Integer, n/2)
        # using Sylvester's method
        P = P_block(T, a, b, b, a)
        m = 4
        for i in 1:lgn
            P = [P zeros(T, m, m); zeros(T, m, m) P]
            m = m * 2
        end
        # mix 4 2-by-2 matrices (with close eigenvalues) into a large nxn matrix.
        B_list = T[102, 1, 1, - 102, 101, 1, 1, 101, 1, 10, 10, 101, 98, 14, 14, 2]
        B = zeros(T, n, n)
        j, k = 1, 5
        for i in 1:(halfn + 1)
            indexend = halfn -1 + i
            list_start = k
            list_end = k + 3

            if list_start > 16 || list_end > 16
                k = 1
                list_start = 1
                list_end = 4
            end
            B[j,j], B[j,indexend], B[indexend, j], B[indexend, indexend] = B_list[list_start:list_end]
            j = j + 1
            k = k + 4
        end
        A = P' * B * P
    end

    return A
end
rosser(::Type{T}, n::Integer) where T = rosser(T, n, rand(1:5), rand(1:5))
rosser(args...) = rosser(Float64, args...)
rosser(::Type, args...) = throw(MethodError(rosser, Tuple(args)))

"""
Matrix with Application in Sampling Theory
==========================================
A nonsymmetric matrix with eigenvalues 0, 1, 2, ... n-1.

*Input options:*

+ [type,] vec: `vec` is a vector with no repeated elements.

+ [type,] dim: `dim` is the dimension of the matrix.
            `vec = [1:dim;]/dim`.

*Groups:* ["eigen"]

*References:*

**L. Bondesson and I. Traat**, A nonsymmetric matrix
            with integer eigenvalues, linear and multilinear algebra, 55(3)
            (2007), pp. 239-247
"""
function sampling(::Type{T}, x::Vector) where T
    n = length(x)
    A = zeros(T, n, n)
    for j = 1:n, i = 1:n
        if i != j
            A[i,j] = x[i] / (x[i] - x[j])
        end
    end
    d = sum(A, dims=2)
    A = A + diagm(0 => d[:])
    return A
end
#
# special probability case
# see:
#   L. Bondesson and I. Traat, A Nonsymmetric Matrix with Integer
#   Eigenvalues, Linear and Multilinear Algebra, 55(3)(2007), pp. 239-247.
#
function sampling(::Type{T}, n::Integer) where T
    p = T[1:n;] / n
    return sampling(T, p)
end
sampling(args...) = sampling(Float64, args...)
sampling(::Type, args...) = throw(MethodError(sampling, Tuple(args)))

"""
Wilkinson Matrix
================
The Wilkinson matrix is a symmetric tridiagonal matrix with pairs
of nearly equal eigenvalues. The most frequently used case
is `matrixdepot("wilkinson", 21)`. The result is of type `Tridiagonal`.

*Input options:*

+ [type,] dim: the dimension of the matrix.

*Groups:* ["symmetric", "eigen"]

*References:*

**J. H. Wilkinson**, Error analysis of direct methods
of matrix inversion, J. Assoc. Comput. Mach., 8 (1961),  pp. 281-330.
"""
function wilkinson(::Type{T}, n::Integer) where T
    if n == 1 # handle 1-d case
        return ones(T, 1, 1)
    end
    m = (n-1)/2
    A = Tridiagonal(ones(T,n-1), abs.(T[-m:m;]), ones(T, n-1))
    return A
end
wilkinson(n::Integer) = wilkinson(Float64, n)

"""
Random Matrix with Element -1, 0, 1
===================================

*Input options:*

+ [type,] row_dim, col_dim, k: `row_dim` and `col_dim` are row and column dimensions,
   `k = 1`: entries are 0 or 1.
   `k = 2`: entries are -1 or 1.
   `k = 3`: entries are -1, 0 or 1.

+ [type,] dim, k: `row_dim = col_dim = dim`.

+ [type,] dim: `k = 1`.

*Groups:* ["random"]
"""
function rando(::Type{T}, m::Integer, n::Integer, k::Integer) where T
    A = Array{T,2}(undef, m, n)
    if k == 1
        copyto!(A, floor.(rand(m,n) .+ .5))
    elseif k == 2
        copyto!(A, 2 * floor.(rand(m,n) .+ .5) .- one(T))
    elseif k == 3
        copyto!(A, round.(3 * rand(m,n) .- 1.5))
    else
        throw(ArgumentError("invalid k value."))
    end
    return A
end
rando(::Type{T}, n::Integer, k::Integer) where T = rando(T, n, n, k)
rando(::Type{T}, n::Integer) where T = rando(T, n, n, 1)
rando(args...) = rando(Float64, args...)
rando(::Type, args...) = throw(MethodError(rando, Tuple(args)))

#
# Pre-multiply by random orthogonal matrix
#
function qmult!(A::Matrix{T}) where T
    n, m = size(A)

    d = zeros(T, n)
    for k = n-1:-1:1

        # generate random Householder transformation
        x = randn(n-k+1)
        s = norm(x)
        sgn = sign(x[1]) + (x[1]==0)
        s = sgn * s
        d[k] = -sgn
        x[1] = x[1] + s
        beta = s * x[1]

        # apply the transformation to A
        y = x'*A[k:n, :];
        A[k:n, :] = A[k:n, :] - x * (y /beta)
    end

    # tidy up signs
    for i=1:n-1
        A[i, :] = d[i] * A[i, :]
    end
    A[n, :] = A[n, :] * sign(randn())
    return A
end

"""
Random Matrix with Pre-assigned Singular Values
===============================================
*Input options:*

+ [type,] row_dim, col_dim, kappa, mode: `row_dim` and `col_dim`
    are the row and column dimensions.
  `kappa` is the condition number of the matrix.
  `mode = 1`: one large singular value.
  `mode = 2`: one small singular value.
  `mode = 3`: geometrically distributed singular values.
  `mode = 4`: arithmetrically distributed singular values.
  `mode = 5`: random singular values with  unif. dist. logarithm.

+ [type,] dim, kappa, mode: `row_dim = col_dim = dim`.

+ [type,] dim, kappa: `mode = 3`.

+ [type,] dim: `kappa = sqrt(1/eps())`, `mode = 3`.

*Groups:* ["illcond", "random"]

*References:*

**N. J. Higham**, Accuracy and Stability of Numerical
Algorithms, second edition, Society for Industrial and Applied Mathematics,
Philadelphia, PA, USA, 2002; sec. 28.3.
"""
function randsvd(::Type{T}, m::Integer, n::Integer, kappa, mode::Integer) where T
    kappa >= 1 || throw(ArgumentError("Condition number must be at least 1."))
    kappa = convert(T, kappa)

    p = min(m,n)
    if p == 1 # handle 1-d case
        return ones(T, 1, 1)*kappa
    end

    if mode == 3
        factor = kappa^(-1/(p-1))
        sigma = factor.^[0:p-1;]
    elseif mode == 4
        sigma = ones(T, p) - T[0:p-1;]/(p-1)*(1 - 1/kappa)
    elseif mode == 5
        sigma = exp.(-rand(p) * log(kappa))
    elseif mode == 2
        sigma = ones(T, p)
        sigma[p] = one(T)/kappa
    elseif mode == 1
        sigma = ones(p)./kappa
        sigma[1] = one(T)
    else
        throw(ArgumentError("invalid mode value."))
    end

    A = zeros(T, m, n)
    A[1:p, 1:p] = diagm(0 => sigma)
    A = qmult!(copy(A'))
    A = qmult!(copy(A'))

    return A
end
randsvd(::Type{T}, n::Integer, kappa, mode) where T = randsvd(T, n, n, kappa, mode)
randsvd(::Type{T}, n::Integer, kappa)  where T= randsvd(T, n, kappa, 3)
randsvd(::Type{T}, n::Integer) where T = randsvd(T, n, sqrt(1/eps(T)))
randsvd(args...) = randsvd(Float64, args...)
randsvd(::Type, args...) = throw(MethodError(randsvd, Tuple(args)))

"""
Random Orthogonal Upper Hessenberg Matrix
=========================================
The matrix is constructed via a product of Givens rotations.

*Input options:*

+ [type,] dim: the dimension of the matrix.

*Groups:* ["random"]

*References:*

**W. B. Gragg**, The QR algorithm for unitary
    Hessenberg matrices, J. Comp. Appl. Math., 16 (1986), pp. 1-8.
"""
function rohess(::Type{T}, n::Integer) where T
    x = rand(n-1)*2*pi
    H = Matrix{T}(I, n, n)
    H[n,n] = sign(randn())
    for i = n:-1:2
        theta = x[i-1]
        c = convert(T, cos(theta))
        s = convert(T, sin(theta))
        H[[i-1; i], :] = [c*H[i-1, :][:]' + s*H[i, :][:]'; -s*H[i-1, :][:]' + c*H[i, :][:]']
    end
    return H
end
rohess(n::Integer) = rohess(Float64, n)

"""
Kac-Murdock-Szego Toeplitz matrix
=================================

*Input options:*

+ [type,] dim, rho: `dim` is the dimension of the matrix, `rho` is a
    scalar such that `A[i,j] = rho^(abs(i-j))`.

+ [type,] dim: `rho = 0.5`.

*Groups:* ["inverse", "illcond", "symmetric", "posdef"]

*References:*

**W. F. Trench**, Numerical solution of the eigenvalue
    problem for Hermitian Toeplitz matrices, SIAM J. Matrix Analysis
    and Appl., 10 (1989), pp. 135-146 (and see the references therein).
"""
function kms(::Type{T}, n::Integer, rho::Number) where T
    A = typeof(rho) <: Complex ? Array{typeof(rho)}(undef, n, n) : Array{T,2}(undef, n, n)
    [A[i,j] = rho^(abs(i-j)) for i = 1:n, j = 1:n]
    if typeof(rho) <: Complex
        A = conj(tril(A, -1)) + triu(A)
    end
    return A
end
kms(::Type{T}, n::Integer) where T = kms(T, n, convert(T, 0.5))
kms(args...) = kms(Float64, args...)
kms(::Type, args...) = throw(MethodError(kms, Tuple(args)))

"""
Wathen Matrix
=============
Wathen Matrix is a sparse, symmetric positive, random matrix
arose from the finite element method. The generated matrix is
the consistent mass matrix for a regular nx-by-ny grid of
8-nodes.

*Input options:*

+ [type,] nx, ny: the dimension of the matrix is equal to
    `3 * nx * ny + 2 * nx * ny + 1`.

+ [type,] n: `nx = ny = n`.

*Groups:* ["symmetric", "posdef", "eigen", "random", "sparse"]

*References:*

**A. J. Wathen**, Realistic eigenvalue bounds for
    the Galerkin mass matrix, IMA J. Numer. Anal., 7 (1987),
    pp. 449-457.
"""
function wathen(::Type{T}, nx::Integer, ny::Integer) where T
    e1 = T[6 -6 2 -8;-6 32 -6 20;2 -6 6 -6;-8 20 -6 32]
    e2 = T[3 -8 2 -6;-8 16 -8 20;2 -8 3 -8;-6 20 -8 16]
    e3 = [e1 e2; e2' e1]/45
    n = 3 * nx * ny + 2 * nx + 2 * ny + 1
    ntriplets = nx * ny * 64
    Irow = zeros(Int, ntriplets)
    Jrow = zeros(Int, ntriplets)
    Xrow = zeros(T, ntriplets)
    ntriplets = 0
    rho = 100 * rand(nx, ny)
    node = zeros(T, 8)

    for j = 1:ny
        for i = 1:nx

            node[1] = 3 * j * nx + 2 * i + 2 * j + 1
            node[2] = node[1] - 1
            node[3] = node[2] - 1
            node[4] = (3 * j - 1) * nx + 2 * j + i - 1
            node[5] = (3 * j - 3) * nx + 2 * j + 2 * i - 3
            node[6] = node[5] + 1
            node[7] = node[5] + 2
            node[8] = node[4] + 1

            em = convert(T, rho[i,j]) * e3

            for krow = 1:8
                for kcol = 1:8
                    ntriplets +=  1
                    Irow[ntriplets] = node[krow]
                    Jrow[ntriplets] = node[kcol]
                    Xrow[ntriplets] = em[krow, kcol]
                end
            end

        end
    end
    return sparse(Irow, Jrow, Xrow, n, n)
end
wathen(::Type{T}, n::Integer) where T = wathen(T, n, n)
wathen(args...) = wathen(Float64, args...)
wathen(::Type, args...) = throw(MethodError(wathen, Tuple(args)))

"""
Golub Matrix
============
Golub matrix is the product of two random unit lower and upper
    triangular matrices respectively. LU factorization without pivoting
    fails to reveal that such matrices are badly conditioned.

*Input options:*

+ [type,] dim: the dimension of the matrix.

*References:*

**D. Viswanath and N. Trefethen**. Condition Numbers of
    Random Triangular Matrices, SIAM J. Matrix Anal. Appl. 19, 564-581,
    1998.
"""
function golub(::Type{T}, n::Integer) where T
    s = 10
    L = Array{T,2}(undef, n, n)
    U = Array{T,2}(undef, n, n)
    if T <: Integer
        [L[i,j] = round_matlab(T, s*randn()) for j = 1:n, i = 1:n]
        [U[i,j] = round_matlab(T, s*randn()) for j = 1:n, i = 1:n]
    else
        [L[i,j] = s*randn() for j = 1:n, i = 1:n]
        [U[i,j] = s*randn() for j = 1:n, i = 1:n]
    end
    L = tril(L, -1) + Matrix{T}(I, n, n)
    U = triu(U, 1) + Matrix{T}(I, n, n)
    return L*U
end
golub(n::Integer) = golub(Float64, n)

"""
Companion Matrix
================
The companion matrix to a monic polynomial
    `a(x) = a_0 + a_1x + ... + a_{n-1}x^{n-1} + x^n`
    is the n-by-n matrix with ones on the subdiagonal and
    the last column given by the coefficients of `a(x)`.

*Input options:*

+ [type,] vec: `vec` is a vector of coefficients.

+ [type,] dim: `vec = [1:dim;]`. `dim` is the dimension of the matrix.
"""
function companion(::Type{T}, v::AbstractVector) where T
    n = length(v)
    A = zeros(T, n, n)
    A[:, end] = v
    for i = 1:n-1
        A[i+1, i] = one(T)
    end
    A
end
companion(::Type{T}, n::Integer) where T = companion(T, [1:n;])
companion(args...) = companion(Float64, args...)
companion(::Type, args...) = throw(MethodError(companion, Tuple(args)))


using MinFEM

function solve_poisson(file_name::String)

  mesh = import_mesh(file_name)

  L = asmLaplacian(mesh)
  M = asmMassMatrix(mesh)

  n=3
  m=2
  f(x) = sin(n*x[1]*pi)*sin(m*x[2]*pi)
  s = evaluateMeshFunction(mesh, f)

  boundary = union(mesh.Boundaries[1001].Nodes,
                   mesh.Boundaries[1002].Nodes,
                   mesh.Boundaries[1003].Nodes,
                   mesh.Boundaries[1004].Nodes)

  pde = PDESystem(A=L, b=M*s, bc=zeros(mesh.nnodes), DI=boundary)

  solve(pde)

  v = ((n*pi)^2 + (m*pi)^2)*pde.state - s
  return sqrt(v'*M*v)

end

@test solve_poisson("test_v2.msh") == solve_poisson("test_v4.msh")
@test solve_poisson("test_v2.msh") < 4e-3


"""
    frft(signal, α)

By entering the signal and the order, we can obtain the Fractional Fourier transform value.

# Example

```julia-repl
julia> frft([1,2,3], 0.5)
0.10046461358821344 - 0.25940948097727745im
3.0746001042331557 + 0.19934938154063286im
1.3156643288728376 - 0.9364535656119107im
```
"""
function frft(signal, α)::Vector{ComplexF64}

    #Ensure the input signal is proper
    isa(signal, AbstractArray) ? nothing : error("Please input a proper signal")

    N = length(signal)
    M = Int64((N-1)/2)

    if rem(N, 2)==0
        error("Signal must be odd")
    end

    ##For α special cases
    #If the order is an integer, the fourier transform would be its n-th order Fourier transform.
    α = mod(α, 4)
    if α == 0
        return signal
    elseif α == 1
        y = fft([signal[M+1:N]; signal[1:M]])/sqrt(N)
        y = [y[M+2 : N]; y[1:M+1]]
        return y
    elseif α == 2
        y = reverse(signal, dims=1)
        return y
    elseif α == 3
        y = fft([signal[M+1:N]; signal[1:M]])*sqrt(N)
        y = [y[M+2 : N]; y[1:M+1]]
        return y
    end

    if α > 2
        signal = reverse(signal, dims=1)
        α = α-2
    end
    if 1.5 < α < 2
        y = fft([signal[M+1:N]; signal[1:M]]) ./sqrt(N)
        y = [y[M+2 : N]; y[1:M+1]]
        α = α-1
    end
    if α < 0.5
        y = fft([signal[M+1:N]; signal[1:M]]) .*sqrt(N)
        y = [y[M+2 : N]; y[1:M+1]]
        α = α+1
    end

    # We need to distinguish **order** and **angle**
    ϕ = α*π/2

    # Set the sampling rate as 3.
    signal = sinc_interp(signal, 3)
    signal = [zeros(ComplexF64, 2*M); signal; zeros(ComplexF64, 2*M)]

    x2 = freq_shear(signal, 2*pi/N*(cot(ϕ)-csc(ϕ))/3^2)
    x3 = time_shear(x2, 2*pi/N*csc(ϕ)/3^2)
    y = freq_shear(x3, 2*pi/N*(cot(ϕ)-csc(ϕ))/3^2)

    y = y[2*M+1:end-2*M]
    y = y[1:3:end]
    y = @. exp(-im*(pi*sign(sin(ϕ))/4-ϕ/2))*y

    return y
end

function freq_shear(x, c)
    x = x[:]
    N = length(x)
    if rem(N, 2) == 0
        error("Signal must be odd")
    end

    M = (N-1)/2
    N = collect(-M:M)

    y = @. x*exp(im*c/2*N^2)
    return y
end

function time_shear(x, c)
    x = x[:]
    N = length(x)

    if rem(N, 2) == 0
        error("Signal must be odd")
    end
    M = Int64((N-1)/2)

    interp = ceil(Int, 2*abs(c)/(2*pi/N))
    xx = sinc_interp(x, interp) ./interp
    n = collect(-2*M:1/interp:2*M)
    z = @. exp(im*c/2*n^2)
    y = conv(xx, z)
    center = (length(y)+1)/2
    y = y[Int64(center-interp*M):Int64(interp):Int64(center+interp*M)]
    y = @. y*sqrt(c/2/pi)
end

"""
    sinc_interp(signal, rate)

```Sinc interpolation``` of **signal** at rate **rate**.

For more details, please refer to [Whittaker-Shannon interpolation](https://en.wikipedia.org/wiki/Whittaker%E2%80%93Shannon_interpolation_formula).
"""
function sinc_interp(x, rate)
    x = x[:]
    N = length(x)
    M = rate*N - rate + 1

    y = zeros(ComplexF64, M)
    y[1:rate:M] = x

    h =  sinc.(collect(-(N-1-1/rate):1/rate:(N-1-1/rate)))
    out = conv(y, h)
    out = out[(rate*N-rate):(end-rate*N+rate+1)]
end


module DiffSolving

export EulerMethod, InstableAlgorithm, EulerCromer, Verlet, VelocityVerlet

using LaTeXStrings, Plots, ProgressBars, Statistics, StatsBase
cd(dirname(@__FILE__))
#First Order ODE_Comparison between Euler method's and Numerical solution

function EulerMethod(func::Function, xᵢₙᵢₜ::Vector{Float64}, tₘᵢₙ::Float64, tₘₐₓ::Float64, h::Float64)
    t_range=collect(range(tₘᵢₙ, tₘₐₓ, step=h))
    Q=zeros(length(t_range), length(xᵢₙᵢₜ))
    Q[1, :]=xᵢₙᵢₜ

    for (i,t) in enumerate(t_range[1:end-1])
        Q[i+1, :] = Q[i, :] + func(t,Q[i, :])*h
    end
    return t_range, Q
end

function InstableAlgorithm(func::Function, xᵢₙᵢₜ::Float64, tₘᵢₙ::Float64, tₘₐₓ::Float64, h::Float64 )
    t_range=collect(range(tₘᵢₙ, tₘₐₓ, step=h))
    Q=zeros(length(t_range))

    Q[1]=xᵢₙᵢₜ
    Q[2]=xᵢₙᵢₜ + h*func(tₘᵢₙ, xᵢₙᵢₜ )
    for i in 2:length(t_range)-1
        Q[i+1] = Q[i-1] + 2*h*func(t_range[i], Q[i])
    end
    return t_range, Q
end

#Second Order ODE
#Simple Harmonic Oscillator

function EulerCromer(func::Function, xᵢₙᵢₜ::Float64, vᵢₙᵢₜ::Float64, tₘᵢₙ::Float64, tₘₐₓ::Float64, h::Float64)
    t=collect(range(tₘᵢₙ, tₘₐₓ, step=h))
    x=zeros(length(t))
    v=zeros(length(t))

    v[1]= vᵢₙᵢₜ
    x[1]= xᵢₙᵢₜ
    for i in 1:length(t)-1
        v[i+1] = v[i] + h * func(t[i], x[i])
        x[i+1] = x[i] + h * v[i+1]
    end

    return t, x, v
end

function Verlet(func::Function, xᵢₙᵢₜ::Float64, vᵢₙᵢₜ::Float64, tₘᵢₙ::Float64, tₘₐₓ::Float64, h::Float64)
    t=collect(range(tₘᵢₙ, tₘₐₓ, step=h))
    x=zeros(length(t))
    v=zeros(length(t))

    v[1]= vᵢₙᵢₜ
    x[1]= xᵢₙᵢₜ
    x[2]= xᵢₙᵢₜ + vᵢₙᵢₜ * h + func(tₘᵢₙ, xᵢₙᵢₜ ) * h^2 *1/2

    for i in 2: length(t)-1
        x[i+1]= 2 * x[i] - x[i-1] + func(t[i], x[i]) * h^2
        v[i+1]= (x[i+1] - x[i])/h
    end
    return t, x, v
end

function VelocityVerlet(func::Function, xᵢₙᵢₜ::Float64, vᵢₙᵢₜ::Float64, tₘᵢₙ::Float64, tₘₐₓ::Float64, h::Float64)
    t= collect(range(tₘᵢₙ, tₘₐₓ, step=h))
    x=zeros(length(t))
    v=zeros(length(t))

    v[1]= vᵢₙᵢₜ
    x[1]= xᵢₙᵢₜ

    for i in 1:length(t)-1
        x[i+1] = x[i] + v[i] * h + func(t[i], x[i]) * h^2 * 1/2
        v[i+1] = v[i] + (func(t[i], x[i]) + func(t[i+1], x[i+1])) * h * 1/2
    end

    return t, x, v
end

end


function score(x, y)
    circles = [
        (1, 10), # inner
        (5, 5),  # middle
        (10, 1), # outer
    ]

    # distance from origin = (0, 0)
    d = sqrt(x^2 + y^2)

    for (radius, score) in circles
        if d <= radius
            return score
        end
    end
    return 0
end