using LinearAlgebra
using Random
using Statistics
using DataFrames
using FreqTables

#:::::::::::::::::::::::::::::::::::::::::::::::::::
# question 1
#:::::::::::::::::::::::::::::::::::::::::::::::::::
using Optim
f(x) = -x[1]^4-10x[1]^3-2x[1]^2-3x[1]-2
minusf(x) = x[1]^4+10x[1]^3+2x[1]^2+3x[1]+2
startval = rand(1)   # random starting value
result = optimize(minusf, startval, BFGS())

#:::::::::::::::::::::::::::::::::::::::::::::::::::
# question 2
#:::::::::::::::::::::::::::::::::::::::::::::::::::
using DataFrames
using CSV
using HTTP
url = "https://raw.githubusercontent.com/OU-PhD-Econometrics/fall-2020/master/ProblemSets/PS1-julia-intro/nlsw88.csv"
df = CSV.read(HTTP.get(url).body,DataFrame)
X = [ones(size(df,1),1) df.age df.race.==1 df.collgrad.==1]
y = df.married.==1

function ols(beta, X, y)
    ssr = (y.-X*beta)'*(y.-X*beta)
    return ssr
end

beta_hat_ols = optimize(b -> ols(b, X, y), rand(size(X,2)), LBFGS(), Optim.Options(g_tol=1e-6, iterations=100_000, show_trace=true))
println(beta_hat_ols.minimizer)

using GLM
bols = inv(X'*X)*X'*y
df.white = df.race.==1
bols_lm = lm(@formula(married ~ age + white + collgrad), df)

#:::::::::::::::::::::::::::::::::::::::::::::::::::
# question 3
#:::::::::::::::::::::::::::::::::::::::::::::::::::
function logit(alpha, X, d)
    loglike = sum([d[i]*log(exp(sum(alpha.*X[i,:]))/(1+exp(sum(alpha.*X[i,:]))))+(1-d[i])*log(1/(1+exp(sum(alpha.*X[i,:])))) for i=1:size(X,1)])
    return loglike
end

startval = rand(size(X,2))
obj = optimize(alpha->-logit(alpha, X, y), startval, LBFGS())
println(obj.minimizer)

#:::::::::::::::::::::::::::::::::::::::::::::::::::
# question 4
#:::::::::::::::::::::::::::::::::::::::::::::::::::
# see Lecture 3 slides for example
res = glm(X, y, Binomial(), LogitLink())
println(res)

#:::::::::::::::::::::::::::::::::::::::::::::::::::
# question 5
#:::::::::::::::::::::::::::::::::::::::::::::::::::
freqtable(df, :occupation) # note small number of obs in some occupations
df = dropmissing(df, :occupation)
df[df.occupation.==8 ,:occupation] .= 7
df[df.occupation.==9 ,:occupation] .= 7
df[df.occupation.==10,:occupation] .= 7
df[df.occupation.==11,:occupation] .= 7
df[df.occupation.==12,:occupation] .= 7
df[df.occupation.==13,:occupation] .= 7
freqtable(df, :occupation) # problem solved

X = [ones(size(df,1),1) df.age df.race.==1 df.collgrad.==1]
y = df.occupation

function mlogit(alpha, X, d)
    loglike = 0
    for i = 1:size(X,1)
        term1 = sum([convert(Float64,d[i]==j)*sum(alpha[:,j].*X[i,:]) for j =1:6])
        term2 = log(1 + sum([exp(sum(alpha[:,j].*X[i,:])) for j =1:6]))
        loglike += term1 - term2
    end
    return loglike
end

#startval = zeros(size(X,2),6)
#startval = rand(size(X,2),6)
startval = 2*rand(size(X,2),6).-1
obj = optimize(alpha->-mlogit(alpha, X, y), startval, LBFGS(), Optim.Options(g_tol=1e-5, iterations=100000))
println(obj.minimizer)


function myf(x)
    return [x[1]+x[2], 5*x[1]^2+3*x[2]-5]
end

function myj(x)
    return [1  1;
            10*x[1]  3]
end

x = newtonraphson(myf, [1,1], myj)


"""
    dirichlet_beta()

 Calculates Dirichlet beta function,
     [https://en.wikipedia.org/wiki/Dirichlet_beta_function](https://en.wikipedia.org/wiki/Dirichlet_beta_function)

## Arguments
* ``s`` `::Number`: it should work for any type of number, but mainly tested for `Complex{Float64}`

## Examples
```jldoctest; setup = :(using Polylogarithms)
julia> dirichlet_beta(1.5)
0.8645026534612017
```
"""
function dirichlet_beta(s::Number)
    β = 4.0^(-s) * ( SpecialFunctions.zeta(s,0.25) - SpecialFunctions.zeta(s,0.75) )
end


using AdFem
using PyPlot
using SparseArrays

function f1func(x,y)
    18.8495559215388*pi^2*sin(pi*x)^2*sin(pi*y)*cos(pi*y) - 6.28318530717959*pi^2*sin(pi*y)*cos(pi*x)^2*cos(pi*y) + pi*sin(pi*y)*cos(pi*x)
end
function f2func(x,y)
    -18.8495559215388*pi^2*sin(pi*x)*sin(pi*y)^2*cos(pi*x) + 6.28318530717959*pi^2*sin(pi*x)*cos(pi*x)*cos(pi*y)^2 + pi*sin(pi*x)*cos(pi*y)
end

m = 50
n = 50
h = 1/n
mmesh = Mesh(m, n, h, degree=2)
ν = 0.5
K = ν*constant(compute_fem_laplace_matrix(mmesh))
B = constant(compute_interaction_matrix(mmesh))
Z = [K -B'
    -B spzero(size(B,1))]

bd = bcnode(mmesh)
bd = [bd; bd .+ mmesh.ndof; 2mmesh.ndof + 1]

F1 = eval_f_on_gauss_pts(f1func, mmesh)
F2 = eval_f_on_gauss_pts(f2func, mmesh)
F = compute_fem_source_term(F1, F2, mmesh)
rhs = [F;zeros(mmesh.nelem)]
Z, rhs = impose_Dirichlet_boundary_conditions(Z, rhs, bd, zeros(length(bd)))
sol = Z\rhs

sess = Session(); init(sess)
S = run(sess, sol)

xy = fem_nodes(mmesh)
x, y = xy[:,1], xy[:,2]
U = @. 2*pi*sin(pi*x)*sin(pi*x)*cos(pi*y)*sin(pi*y)
figure(figsize=(12,5))
subplot(121)
visualize_scalar_on_fem_points(U, mmesh)
title("Reference")
subplot(122)
visualize_scalar_on_fem_points(S[1:mmesh.nnode], mmesh)
title("Computed")
savefig("stokes1.png")

U = @. -2*pi*sin(pi*x)*sin(pi*y)*cos(pi*x)*sin(pi*y)
figure(figsize=(12,5))
subplot(121)
visualize_scalar_on_fem_points(U, mmesh)
title("Reference")
subplot(122)
visualize_scalar_on_fem_points(S[mmesh.ndof+1:mmesh.ndof+mmesh.nnode], mmesh)
title("Computed")
savefig("stokes2.png")

xy = fvm_nodes(mmesh)
x, y = xy[:,1], xy[:,2]
p = @. sin(pi*x)*sin(pi*y)
figure(figsize=(12,5))
subplot(121)
visualize_scalar_on_fvm_points(p, mmesh)
title("Reference")
subplot(122)
visualize_scalar_on_fvm_points(S[2mmesh.ndof+1:end], mmesh)
title("Computed")
savefig("stokes3.png")


include("../aimajulia.jl");

using Test;

using Main.aimajulia;

using Main.aimajulia.utils;

using LinearAlgebra;

#The following learning tests are from the aima-python doctests

@test (repr(euclidean_distance([1, 2], [3, 4])) == "2.8284271247461903");

@test (repr(euclidean_distance([1, 2, 3], [4, 5, 6])) == "5.196152422706632");

@test (repr(euclidean_distance([0, 0, 0], [0, 0, 0])) == "0.0");

@test (root_mean_square_error([2, 2], [2, 2]) == 0);

@test (root_mean_square_error([0, 0], [0, 1]) == sqrt(0.5));

@test (root_mean_square_error([1, 0], [0, 1]) == 1);

@test (root_mean_square_error([0, 0], [0, -1]) == sqrt(0.5));

@test (root_mean_square_error([0, 0.5], [0, -0.5]) == sqrt(0.5));

@test (manhattan_distance([2, 2], [2, 2]) == 0);

@test (manhattan_distance([0, 0], [0, 1]) == 1);

@test (manhattan_distance([1, 0], [0, 1]) == 2);

@test (manhattan_distance([0, 0], [0, -1]) == 1);

@test (manhattan_distance([0, 0.5], [0, -0.5]) == 1);

@test (mean_boolean_error([1, 1], [0, 0]) == 1)

@test (mean_boolean_error([0, 1], [1, 0]) == 1)

@test (mean_boolean_error([1, 1], [0, 1]) == 0.5)

@test (mean_boolean_error([0, 0], [0, 0]) == 0)

@test (mean_boolean_error([1, 1], [1, 1]) == 0)

@test (mean_error([2, 2], [2, 2]) == 0);

@test (mean_error([0, 0], [0, 1]) == 0.5);

@test (mean_error([1, 0], [0, 1]) ==  1);

@test (mean_error([0, 0], [0, -1]) ==  0.5);

@test (mean_error([0, 0.5], [0, -0.5]) == 0.5);

@test (gaussian(1,0.5,0.7) == 0.6664492057835993);

@test (gaussian(5,2,4.5) == 0.19333405840142462);

@test (gaussian(3,1,3) == 0.3989422804014327);

iris_dataset = DataSet(name="iris", examples="./aima-data/iris.csv", exclude=[4]);

@test (iris_dataset.inputs == [1, 2, 3]);

iris_dataset = DataSet(name="iris", examples="./aima-data/iris.csv");
means_dict, deviations_dict = find_means_and_deviations(iris_dataset);

@test (means_dict["setosa"][1] == 5.006);

@test (means_dict["versicolor"][1] == 5.935999999999999);

@test (means_dict["virginica"][1] == 6.587999999999999);

@test (deviations_dict["setosa"][1] == 0.3524896872134513);

@test (deviations_dict["versicolor"][1] == 0.5161711470638634);

@test (deviations_dict["virginica"][1] == 0.6358795932744321);

cpd = CountingProbabilityDistribution();

for i in 1:10000
    add(cpd, rand(RandomDeviceInstance, ["1", "2", "3", "4", "5", "6"]));
end

probabilities = collect(cpd[n] for n in ("1", "2", "3", "4", "5", "6"));

@test ((1.0/7.0) <= reduce(min, probabilities) <= reduce(max, probabilities) <= (1.0/5.0));

zoo_dataset = DataSet(name="zoo", examples="./aima-data/zoo.csv");

pl = PluralityLearner(zoo_dataset);

@test (predict(pl, [1, 0, 0, 1, 0, 0, 0, 1, 1, 1, 0, 0, 4, 1, 0, 1]) == "mammal");

iris_dataset = DataSet(name="iris", examples="./aima-data/iris.csv");

# Naive Discrete Model
nbdm = NaiveBayesLearner(iris_dataset, continuous=false);

@test (predict(nbdm, [5, 3, 1, 0.1]) == "setosa");

@test (predict(nbdm, [6, 3, 4, 1.1]) == "versicolor");

@test (predict(nbdm, [7.7, 3, 6, 2]) == "virginica");

# Naive Continuous Model
nbcm = NaiveBayesLearner(iris_dataset, continuous=true);

@test (predict(nbcm, [5, 3, 1, 0.1]) == "setosa");

@test (predict(nbcm, [6, 5, 3, 1.5]) == "versicolor");

@test (predict(nbcm, [7, 3, 6.5, 2]) == "virginica");

# Naive Conditional Probability Model
d1 = CountingProbabilityDistribution(vcat(fill('a', 50), fill('b', 30), fill('c', 15)));
d2 = CountingProbabilityDistribution(vcat(fill('a', 30), fill('b', 45), fill('c', 20)));
d3 = CountingProbabilityDistribution(vcat(fill('a', 20), fill('b', 20), fill('c', 35)));
nbsm = NaiveBayesLearner(Dict([(("First", 0.5), d1), (("Second", 0.3), d2), (("Third", 0.2), d3)]), simple=true);

@test (predict(nbsm, "aab") == "First");

@test (predict(nbsm, ['b', 'b']) == "Second");

@test (predict(nbsm, "ccbcc") == "Third");

iris_dataset = DataSet(name="iris", examples="./aima-data/iris.csv");

k_nearest_neighbors = NearestNeighborLearner(iris_dataset, 3);

@test (predict(k_nearest_neighbors, [5, 3, 1, 0.1]) == "setosa");

@test (predict(k_nearest_neighbors, [6, 5, 3, 1.5]) == "versicolor");

@test (predict(k_nearest_neighbors, [7.5, 4, 6, 2]) == "virginica");

iris_dataset = DataSet(name="iris", examples="./aima-data/iris.csv");

# Truncated SVD
test_epsilon = 1e-9

# Test for consistent properties of the SVD, recall that we randomize initial values during computation
for test_iteration in 1:1000
    U, S, V = svd([1 5 9 13; 2 6 10 14; 3 7 11 15; 4 8 12 16]);
    ev_m, ev_n, singular_values = truncated_svd([1 5 9 13; 2 6 10 14; 3 7 11 15; 4 8 12 16]);

    @test (all(i->(abs(i) < test_epsilon), singular_values - S[1:2]));

    # A * v = eigenvalue * v
    @test (all(i->(abs(i) < test_epsilon),
                (ev_n[1]*singular_values[1])
                - [1 5 9 13; 2 6 10 14; 3 7 11 15; 4 8 12 16]*ev_m[1]));
    @test (all(i->(abs(i) < test_epsilon),
                (ev_n[2]*singular_values[2])
                - [1 5 9 13; 2 6 10 14; 3 7 11 15; 4 8 12 16]*ev_m[2]));

    # check if returned eigenvectors applies to the transpose of our original matrix
    @test (all(i->(abs(i) < test_epsilon),
                (ev_m[1]*singular_values[1])
                - [1 2 3 4; 5 6 7 8; 9 10 11 12; 13 14 15 16]*ev_n[1]));
    @test (all(i->(abs(i) < test_epsilon),
                (ev_m[2]*singular_values[2])
                - [1 2 3 4; 5 6 7 8; 9 10 11 12; 13 14 15 16]*ev_n[2]));

    #A=U*S*V^T
    @test (all(i->(abs(i) < test_epsilon),
                ([1.0 5.0 9.0 13.0; 2.0 6.0 10.0 14.0; 3.0 7.0 11.0 15.0; 4.0 8.0 12.0 16.0]
                - (hcat(ev_n[1], ev_n[2]) * Diagonal(singular_values) * transpose(hcat(ev_m[1], ev_m[2]))))));
end

# aima-python doctests for truncated_svd()
@test (all(i->(abs(i) < test_epsilon),
            (truncated_svd([17 0; 0 11])[3] - [17.0, 11.0])));

@test (all(i->(abs(i) < test_epsilon),
            (truncated_svd([17 0; 0 -34])[3] - [34.0, 17.0])));

@test (all(i->(abs(i) < test_epsilon),
            (truncated_svd([1 0 0 0 2; 0 0 3 0 0; 0 0 0 0 0; 0 2 0 0 0])[3] - [3.0, sqrt(5)])));

@test (all(i->(abs(i) < test_epsilon),
            (truncated_svd([3 2 2; 2 3 -2])[3] - [5.0, 3.0])));

# Decision Tree
dtl = DecisionTreeLearner(iris_dataset);

@test (predict(dtl, [5, 3, 1, 0.1]) == "setosa");

@test (predict(dtl, [6, 5, 3, 1.5]) == "versicolor");

@test (predict(dtl, [7.5, 4, 6, 2]) == "virginica");

function test_rf_predictions(ex1_results::AbstractVector, ex2_results::AbstractVector, ex3_results::AbstractVector)
    local rf::RandomForest = RandomForest(iris_dataset);
    push!(ex1_results, (predict(rf, [5, 3, 1, 0.1]) == "setosa"));
    push!(ex2_results, (predict(rf, [6, 5, 3, 1]) == "versicolor"));
    push!(ex3_results, (predict(rf, [7.5, 4, 6, 2]) == "virginica"));
    nothing;
end

setosa_results = Array{Bool, 1}();
versicolor_results = Array{Bool, 1}();
virginica_results = Array{Bool, 1}();
# Run test_rf_predictions() 1000 times.
println("@time for i in 1:1000\n\ttest_rf_predictions(setosa_results, versicolor_results, virginica_results);\nend");
@time for i in 1:1000
    test_rf_predictions(setosa_results, versicolor_results, virginica_results);
end

setosa_results_count = count((function(b::Bool)
                                    return b;
                                end), setosa_results);
versicolor_results_count = count((function(b::Bool)
                                    return b;
                                end), versicolor_results);
virginica_results_count = count((function(b::Bool)
                                    return b;
                                end), virginica_results);

# lowest setosa_results_count result previously obtained was 970
@test (setosa_results_count >= 960);

# lowest versicolor_results_count result previously obtained was 959
@test (versicolor_results_count >= 950);

# lowest virginica_results_count result previously obtained was 996
@test (virginica_results_count >= 990);

println();
println("setosa assert count (out of 1000): ", setosa_results_count);
println("setosa assertion failure rate: approximately ", Float64(1000 - setosa_results_count)/10.0, "%");
println("versicolor assert count (out of 1000): ", versicolor_results_count);
println("versicolor assertion failure rate: approximately ", Float64(1000 - versicolor_results_count)/10.0, "%");
println("virginica assert count (out of 1000): ", virginica_results_count);
println("virginica assertion failure rate: approximately ", Float64(1000 - virginica_results_count)/10.0, "%");
println();

weights = random_weights(-0.5, 0.5, 10);

@test (length(weights) == 10);

@test (all(((weight >= -0.5) && (weight <= 0.5)) for weight in weights));

iris_dataset = DataSet(name="iris", examples="./aima-data/iris.csv");

# The DataType of the example classification must match the eltype of the classes array.

classes = map(SubString{String}, ["setosa", "versicolor", "virginica"]);

classes_to_numbers(iris_dataset, classes);

nnl = NeuralNetworkLearner(iris_dataset, hidden_layers_sizes=[5], learning_rate=0.15, epochs=75);

neural_network_learner_score = aimajulia.grade_learner(nnl,
                                                        [([5, 3, 1, 0.1], 1),
                                                        ([5, 3.5, 1, 0], 1),
                                                        ([6, 3, 4, 1.1], 2),
                                                        ([6, 2, 3.5, 1], 2),
                                                        ([7.5, 4, 6, 2], 3),
                                                        ([7, 3, 6, 2.5], 3)]);

println("neural network learner score (out of 1.0): ", neural_network_learner_score);

# Allow up to 2 failed tests.
@test (neural_network_learner_score >= (2/3));

neural_network_learner_error_ratio = aimajulia.error_ratio(nnl, iris_dataset);

println("neural network learner error ratio: ", (neural_network_learner_error_ratio * 100), "%");
println();

# NeuralNetworkLearner previously had an error ratio of 0.33333333333333337.
@test (neural_network_learner_error_ratio < 0.40);

iris_dataset = DataSet(name="iris", examples="./aima-data/iris.csv");

classes_to_numbers(iris_dataset, nothing);

pl = PerceptronLearner(iris_dataset);

perceptron_learner_score = aimajulia.grade_learner(pl,
                                                    [([5, 3, 1, 0.1], 1),
                                                    ([5, 3.5, 1, 0], 1),
                                                    ([6, 3, 4, 1.1], 2),
                                                    ([6, 2, 3.5, 1], 2),
                                                    ([7.5, 4, 6, 2], 3),
                                                    ([7, 3, 6, 2.5], 3)]);

println("perceptron learner score (out of 1.0): ", perceptron_learner_score);

# Allow up to 3 failed tests.
@test (perceptron_learner_score >= (1/2));

perceptron_learner_error_ratio = aimajulia.error_ratio(pl, iris_dataset);

println("perceptron learner error ratio: ", (perceptron_learner_error_ratio * 100), "%");
println();

@test (perceptron_learner_error_ratio < 0.40);

@test (weighted_mode("abbaa", [1, 2, 3, 1, 2]) == "b");

@test (weighted_mode(["a", "b", "b", "a", "a"], [1, 2, 3, 1, 2]) == "b");

@test (weighted_replicate(["A", "B", "C"], [1, 2, 1], 4) == ["A", "B", "B", "C"]);


function acqr_kfilter(y,a,c,q,r,x10,P10)
	#
	# Kalman filter for model
	#
	# x_{t+1} = a*x_{t} + w_{t}
	# y_{t}   = c*x_{t} + v_{t}
	#
	# cov(w_{t},v_{t}) = [q 0;0 r]
	#
	# javier.cara@upm.es, 2020-05
	#

	nt = length(y)

	# allocation
	xtt = zeros(nt)
	Ptt = zeros(nt)
	xtt1 = zeros(nt+1)
	Ptt1 = zeros(nt+1)
	et = zeros(nt)
	St = zeros(nt)
	Kt = zeros(nt)
	loglik = 0.0

	# Filter
	xtt1[1] = x10
	Ptt1[1] = P10
	for t in 1:nt
		#  innovations
		et[t] = y[t] - c*xtt1[t]
		St[t] = c^2*Ptt1[t] + r # et variance

		# Kalman gain
		Kt[t] = Ptt1[t]*c/St[t] # kalman gain

		# filtered values
		xtt[t] = xtt1[t] + Kt[t]*et[t]
		Ptt[t] = (1 - Kt[t]*c)*Ptt1[t]

		# one-step ahead prediction
		xtt1[t+1] = a*xtt[t]
		Ptt1[t+1] = a^2*Ptt[t] + q

		# likelihood
		#l0 = et[t]^2/St[t] # typeof(l0) = Array{Float64,1}
		loglik = loglik + log(St[t]) + et[t]^2/St[t]
	end

	loglik =  - nt/2*log(2*pi) - 0.5*loglik

	return xtt,Ptt,xtt1,Ptt1,et,St,Kt,loglik

end


function nlp_002_010(
    optimizer,
    objective_tol,
    primal_tol,
    dual_tol,
    termination_target = TERMINATION_TARGET_LOCAL,
    primal_target = PRIMAL_TARGET_LOCAL,
)
    # Test Goals:
    # - all have start values
    # - non-linear constraints without objective
    # - functions log

    model = Model(optimizer)

    @variable(model, x, start = 1)
    @variable(model, y, start = -1.12)

    @NLconstraint(model, y == log(x) - 0.1)
    @NLconstraint(model, x == cos(y)^2 + 1.5)

    optimize!(model)

    check_status(model, FEASIBLE_PROBLEM, termination_target, primal_target)
    check_objective(model, 0.0, tol = objective_tol)
    return check_solution(
        [x, y],
        [2.1285148443189033, 0.6554244804634232],
        tol = primal_tol,
    )
end


module FractionalCalculus

using QuadGK
using SpecialFunctions: gamma
using LinearAlgebra, InvertedIndices, SpecialMatrices

include("./Derivative/Derivative.jl")
include("./Integral/Integral.jl")

export FracDiffAlg, FracIntAlg
export fracdiff, fracint

# Export fractional derivative releating API
export Caputo, GL, RLDiff, Hadamard

# Caputo sense fractional derivative
export Caputo_Direct, Caputo_Direct_First_Diff_Known, Caputo_Direct_First_Second_Diff_Known
export Caputo_Piecewise
export Caputo_Diethelm
export Caputo_High_Precision

# Grünwald Letnikov sense fractional derivative
export GL_Direct, GL_Multiplicative_Additive, GL_Lagrange_Three_Point_Interp, GL_High_Precision

# Riemann Liouville sense fractional derivative
export RLDiff_Approx, RLDiff_Matrix, RL_Linear_Spline_Interp

# Hadamard sense fractional derivative
export Hadamard_LRect, Hadamard_RRect, Hadamard_Trap

# Riesz sense fractional derivative
export Riesz_Symmetric

# Macros for convenient computing
export @fracdiff, @semifracdiff

#Export fractional integral relating API
export RLInt
export RL_Direct, RL_Direct_First_Diff_Known, RL_Piecewise
export RLInt_Approx, RL_LinearInterp, RLInt_Matrix
export RLInt_Simpson, RLInt_Trapezoidal, RLInt_Rectangular, RLInt_Cubic_Spline_Interp

# Macros for convenient computing
export @fracint, @semifracint

end


# # Heat Equation
#
# This example demonstrates how to combine `OrdinaryDiffEq` with `DiffEqOperators` to solve a time dependent PDE.
# We consider the heat equation on the unit interval, with dirichlet boundary conditions:
# ∂ₜu = Δu
# u(x=0,t)  = a
# u(x=1,t)  = b
# u(x, t=0) = u₀(x)
#
# For `a = b = 0` and `u₀(x) = sin(2πx)` a solution is given by:
u_analytic(x, t) = sin(2*π*x) * exp(-t*(2*π)^2)
#
# We want to reproduce it numerically
#
using DiffEqOperators, OrdinaryDiffEq

nknots = 100
h = 1.0/(nknots+1)
knots = range(h, step=h, length=nknots)
ord_deriv = 2
ord_approx = 2

const Δ = CenteredDifference(ord_deriv, ord_approx, h, nknots)
const bc = Dirichlet0BC(Float64)

t0 = 0.0
t1 = 0.03
u0 = u_analytic.(knots, t0)

step(u,p,t) = Δ*bc*u
prob = ODEProblem(step, u0, (t0, t1))
alg = KenCarp4()
sol = solve(prob, alg)

using Test
@test u_analytic.(knots, t1) ≈ sol[end] rtol=1e-3


# # Maximum Cut Problem
#
# We are interested in solving an approximation to the maximum cut problem using semidefinite programming.
# Consider the graph $G(V,E)$ with weights $w_{ij}$ shown below:

## create the weighted adjacency matrix of the graph and plot the graph
using LinearAlgebra, Plots, GraphRecipes
n = 4;
W = zeros(n, n);
W[1, 2] = 1; W[1, 4] = 8;
W[2, 3] = 2; W[2, 4] = 10;
W[3, 4] = 6;
W = Symmetric(W)
graphplot(W, names = 1:n, edgelabel = W,  x = [0; 1; 1; 0], y = [1; 1; 0; 0], fontsize = 12, nodeshape =:circle)

# The maximum cut problem tries to find a cut or a partition of the graph's vertices into two complementary sets $S$ and $\bar{S}$ such that the total weight of the edges between the two sets is maximized. For this small graph the problem is trivial. The optimal solution is $S = \{1,2,3 \}$ and $\bar{S}=\{4\}$.
# Formally, this problem can be written as a mixed-integer optimisation problem:
# $$
# \begin{array}{ll} \text{maximize} &  \frac{1}{2} \sum_{i < j} w_{ij}(1 - y_i y_j)\\
# \text{subject to} &  y_i \in \, \{-1, 1 \}, \quad \forall i \in V,
# \end{array}
# $$
# where $y_i = 1$ indicates that $v_i \in S$ and $y_i = -1$ indicates that $v_i \in \bar{S}$. This problem is of interest in the field of integrated circuit design, where one tries to minimize the number of cross-layer connections in a circuit under layout constraints.

# For more complicated graphs this problem quickly becomes hard to solve to optimality and in fact is known to be NP-hard. For this example we are interested in the randomized approximation algorithm that relaxes the problem to an SDP and was devised in Goemans and Williamson (1995) \[1\] (see for more details).

# The approach can be divided into three steps:
# 1. Solve a relaxed SDP to obtain $Y^*$
# 2. Recover an approximate solution $V$ via a Cholesky factorisation $Y^* = V^\top V$
# 3. Round the approximate solution using a random vector $r$ from the unit sphere.
# The authors showed that this approach guarantees a solution of at least $0.87856$ times the optimal solution.

# ## Solving the primal SDP
# Before we formulate the SDP, let's compute the Laplacian matrix $L$:
using LinearAlgebra, Random, SparseArrays, COSMO, JuMP

rng = Random.MersenneTwister(1)

## Compute the Laplacian matrix of the graph
L = diagm(0 => W * ones(n)) - W;

# Given the Laplacian matrix $L$, we are looking for the optimal $Y^*$ that solves the following SDP:
# $$
# \begin{array}{ll} \text{maximize} &  \frac{1}{4} \langle L, Y \rangle \\
# \text{subject to} & Y_{ii} = 1 \quad \text{for } i = 1,\dots,n\\
#                    & Y \in \mathbf{S}_{+}^n.
# \end{array}
# $$
# Notice that the solution $Y^*$ can be viewed as a correlation matrix. Let's solve the problem using COSMO and JuMP.

#-
model = JuMP.Model(COSMO.Optimizer);
@variable(model, Y[1:n, 1:n], PSD);
@objective(model, Max, 1 / 4 * dot(L, Y));
@constraint(model, [i = 1:n], Y[i, i] == 1.);
JuMP.optimize!(model)

#-
Yopt = JuMP.value.(Y);
obj_val = JuMP.objective_value(model)

# ## Solving the dual SDP
# Notice that the decision matrix $Y$ is generally dense (as correctly classified in the solver output above). Therefore, we won't be able to utilize COSMO's chordal decomposition features. However, assuming strong duality, it turns out that we can also solve the dual problem, which is given by:
# $$
# \begin{array}{ll} \text{minimize} &  \sum_i \gamma_i \\
# \text{subject to} & \text{diag}(\gamma) - \frac{1}{4} L = S \\
#                   & S \in \mathbf{S}_{+}^n.
# \end{array}
# $$
# As you can see, the matrix $S$ is constrained to have zeros in places specified by the graph (Laplacian). Therefore, COSMO can try to decompose this SDP (see solver output) and speed up its algorithm:

#-
model_dual = JuMP.Model(optimizer_with_attributes(COSMO.Optimizer, "complete_dual" => true));
@variable(model_dual, γ[1:n]);
@objective(model_dual, Min,  sum(γ));
@constraint(model_dual, lmi, Symmetric(-1/4 * L + diagm(γ)) in JuMP.PSDCone());
JuMP.optimize!(model_dual)
obj_val = JuMP.objective_value(model_dual)

# The primal variable $Y^*$ can be recovered from the dual variable associated with the LMI-constraint:
#-
Yopt = dual(lmi)
# To get the correct positive semidefinite dual variable, we have to enable PSD completion of the dual variable in COSMO. Now, that we have a solution we can perform the remaining steps in the approximation algorithm.

# ## Cholesky factorisation of Y
# Compute the Cholesky factorisation of $Y = V^\top V$ to find the unit vectors $v_1, \dots, v_n$.

#-
factor = cholesky(Yopt, Val(true); check = false);
V = Matrix((factor.P * factor.L)');
## normalize columns
for i in 1:n
    V[:, i] ./= norm(V[:, i]);
end

# ## Rounding the approximate solution
# It remains to round the unit vectors $v_i$ using a random vector $r$ with each component drawn from $\mathcal{U}(0, 1)$, to obtain the $y_i$'s:

#-
r = rand(rng, n)
r /= norm(r, 2)
y = ones(n)
for i in 1:n
  dot(r, V[:, i]) <= 0 && (y[i] = -1)
end
y
# For larger graphs this rounding step could be repeated several times to improve the rounding.
# As expected $S = \{1, 2, 3\}$ and $\bar{S}=\{ 4\}$.
# ## References
# [1] Goemans and Williamson - Improved Approximation Algorithms for Maximum Cut and Satisfiability Problems Using Semidefinite Programs (1995)

# [2] Post-processing code steps 2 and 3 from this [JuMP example](https://github.com/jump-dev/JuMP.jl/blob/master/examples/max_cut_sdp.jl)


f(u,p,t) = 0.98u

using DifferentialEquations
f(u,p,t) = 0.98u
u0 = 1.0
tspan = (0.0,1.0)
prob = ODEProblem(f,u0,tspan)

sol = solve(prob)

using Plots; gr()
plot(sol)

plot(sol,linewidth=5,title="Solution to the linear ODE with a thick line",
     xaxis="Time (t)",yaxis="u(t) (in μm)",label="My Thick Line!") # legend=false

plot!(sol.t, t->1.0*exp(0.98t),lw=3,ls=:dash,label="True Solution!")

sol.t

sol.u

[t+u for (u,t) in tuples(sol)]

sol

sol(0.45)

sol = solve(prob,abstol=1e-8,reltol=1e-8)

plot(sol)
plot!(sol.t, t->1.0*exp(0.98t),lw=3,ls=:dash,label="True Solution!")

sol = solve(prob,saveat=0.1)

sol = solve(prob,saveat=[0.2,0.7,0.9])

sol = solve(prob,dense=false)

sol = solve(prob,save_everystep=false)

sol = solve(prob,save_everystep=false,save_start = false)

sol = solve(prob,alg_hints=[:stiff])

sol = solve(prob,Tsit5(),reltol=1e-6)

function lorenz!(du,u,p,t)
    σ,ρ,β = p
    du[1] = σ*(u[2]-u[1])
    du[2] = u[1]*(ρ-u[3]) - u[2]
    du[3] = u[1]*u[2] - β*u[3]
end

u0 = [1.0,0.0,0.0]

p = (10,28,8/3) # we could also make this an array, or any other type!

tspan = (0.0,100.0)
prob = ODEProblem(lorenz!,u0,tspan,p)

sol = solve(prob)

sol.t[10],sol[10]

sol[2,10]

A = Array(sol)

plot(sol)

plot(sol,vars=(1,2,3))

plot(sol,vars=(1,2,3),denseplot=false)

plot(sol,vars=(0,2))

function lotka_volterra!(du,u,p,t)
  du[1] = p[1]*u[1] - p[2]*u[1]*u[2]
  du[2] = -p[3]*u[2] + p[4]*u[1]*u[2]
end

using ParameterizedFunctions
lv! = @ode_def LotkaVolterra begin
  dx = a*x - b*x*y
  dy = -c*y + d*x*y
end a b c d

u0 = [1.0,1.0]
p = (1.5,1.0,3.0,1.0)
tspan = (0.0,10.0)
prob = ODEProblem(lv!,u0,tspan,p)
sol = solve(prob)
plot(sol)

lv!.Jex

A  = [1. 0  0 -5
      4 -2  4 -3
     -4  0  0  1
      5 -2  2  3]
u0 = rand(4,2)
tspan = (0.0,1.0)
f(u,p,t) = A*u
prob = ODEProblem(f,u0,tspan)
sol = solve(prob)

sol[3]

big_u0 = big.(u0)

prob = ODEProblem(f,big_u0,tspan)
sol = solve(prob)

sol[1,3]

prob = ODEProblem(f,big_u0,big.(tspan))
sol = solve(prob)

using StaticArrays
A  = @SMatrix [ 1.0  0.0 0.0 -5.0
                4.0 -2.0 4.0 -3.0
               -4.0  0.0 0.0  1.0
                5.0 -2.0 2.0  3.0]
u0 = @SMatrix rand(4,2)
tspan = (0.0,1.0)
f(u,p,t) = A*u
prob = ODEProblem(f,u0,tspan)
sol = solve(prob)

sol[3]

using DiffEqTutorials
DiffEqTutorials.tutorial_footer(WEAVE_ARGS[:folder],WEAVE_ARGS[:file])


function romberg(f::Function, a::Number, b::Number, nl::Int)
  #= Numerical Integration of functions using Romberg's method.
  Keyword arguments:
  f -- Anonymous functon to integrate
  a -- Lower limit of integration
  b -- Upper limit of integration
  nl -- The level of integration to go to.

  Returns:
  The result of the integration of f between a & b
  =#

  intTable = zeros(nl, nl); # Empty matrix to use as div diff table

  # Calculate the first column of the div diff table using the trapezium rule.
  h = (b - a);
  for k = 1:nl
    intTable[k,1] = 0.5 * h * (f(a) + 2*sum(f([a+h:h:b-h])) + f(b));
    h /= 2;
  end

  # Calculate the remaining elements using Romberg's method.
  for k = 2:nl, j = nl-k+1:-1:1
    intTable[j, k] = ((4^(k-1))intTable[j+1, k-1] - intTable[j, k-1])/((4^(k-1)) - 1);
  end

  return intTable[1, end];
end

function romberg(intEstimates::Array{Float64,1})
  #= Estiamte integrals from sets of data using the Romberg Method.

  Keyword arguments:
  intEstimates -- An array of initial estimates of integrals

  Returns:
    A tuple containing the integral and the relative error in the estimate
  =#
  intTable = zeros(length(intEstimates), length(intEstimates));
  relativeErrorTable = zeros(size(intEstimates));

  intTable[:, 1] = intEstimates;

  for k = 2:size(intTable)[1]
    for j = size(intTable)[1]-k+1:-1:1
      intTable[j, k] = ((4^(k-1))intTable[j+1, k-1] - intTable[j, k-1])/((4^(k-1)) - 1);
    end
  end

  for k = 2:size(intTable)[1]
    relativeErrorTable[k, 1] = 100 * abs((intTable[1, k] - intTable[2, k-1]) / intTable[1, k]);
  end

  return (intTable[1, end], relativeErrorTable[1, end]);
end


function calc_seat_id(pass)
    digits = map(c -> c == 'F' || c == 'L' ? '0' : '1', pass)
    row = parse(Int64, digits[1:7]; base = 2)
    column = parse(Int64, digits[8:end]; base = 2)
    row * 8  + column
end

function seat_ids_on_list(boarding_passes)
    ids = Set{Int64}()
    for pass in boarding_passes
        push!(ids, calc_seat_id(pass))
    end
    ids
end

function find_missing_id(boarding_passes)
    existing_ids = seat_ids_on_list(boarding_passes)
    for row in 1:126, column in 0:7
        seat_id = row * 8 + column
        if seat_id ∉ existing_ids && (seat_id - 1) ∈ existing_ids && (seat_id + 1) ∈ existing_ids
            return seat_id
        end
    end
end

function main()
    lines = readlines("../resources/input_05.txt")
    result = find_missing_id(lines)
    println(result)
end

main()


using CompScienceMeshes
using SauterSchwabQuadrature

pI = point(1,5,3)
pII = point(2,5,3)
pIII = point(7,1,0)
pIV = point(5,1,-3)

Sourcechart = simplex(pII,pI,pIII)
Testchart = simplex(pII,pI,pIV)

Accuracy = 12

function integrand(x,y)
			return(((x-pI)'*(y-pII))*exp(-im*1*norm(x-y))/(4pi*norm(x-y)))
end

result = sauterschwabintegral(Sourcechart, Testchart, integrand, Accuracy, Accuracy)
println(result)

#=For those who want to test the sauterschwab_nonparameterized() function,
may uncomment the following five lines=#

#sourcechart = simplex(pI,pIII,pII)
#testchart = simplex(pI,pIV,pII)
#ce = CommonEdge(Accuracy)
#result2 = sauterschwab_nonparameterized(sourcechart, testchart, integrand, ce)
#println(result2)

#=The first argument and the third argument of both simplex() functions are
equal, hence the required conditions are fulfilled. The user may also compare
the two results, and see that both are equal=#


# ------------------------------------------------------------------------------------------
# # Basics: Arithmetic, Assignment, and Strings
#
# Topics:
# 1. Infix arithmetic operators
# 2. Numeric literals
# 3. Comparisons
# 4. Assignments
# 5. Strings
# ------------------------------------------------------------------------------------------

# ------------------------------------------------------------------------------------------
# ## Arithmetic should largely be familiar
# ------------------------------------------------------------------------------------------

3 + 7 # addition

10 - 3 # subtraction

20 * 5 # multiplication

100 / 10 # division

10 ^ 2 # exponentiation

101 % 2 # remainder (modulus)

sqrt(2) # square root

√2 # Unicode to the rescue: \sqrt + TAB

# ------------------------------------------------------------------------------------------
# Note that dividing two integers yields a floating point number. There are two additional
# operators that may be helpful here:
# ------------------------------------------------------------------------------------------

10 / 6

10 ÷ 6 # \div TAB or the `div` function

div(10, 6)

10 // 6

# ------------------------------------------------------------------------------------------
# ### Numbers: Many different ways to write the number forty-two
# ------------------------------------------------------------------------------------------

fortytwos = (42, 42.0, 4.20e1, 4.20f1, 84//2, 0x2a)

for x in fortytwos
    show(x)
    println("\tisa $(typeof(x))")
end

# ------------------------------------------------------------------------------------------
# ### Bitwise arithmetic
# ------------------------------------------------------------------------------------------

0x2a & 0x70 # AND

0x2a | 0x70 # OR

42 & 112

0b0010 << 2 # == 0b1000

# ------------------------------------------------------------------------------------------
# ### Logical operators
# ------------------------------------------------------------------------------------------

false && true # AND

false || true # OR

# ------------------------------------------------------------------------------------------
# Note that they "short-circuit!"
# ------------------------------------------------------------------------------------------

x = -42
x > 0 || error("x must be positive")

# ------------------------------------------------------------------------------------------
# ### Comparisons
# ------------------------------------------------------------------------------------------

1 == 1.0 # Equality

1 === 1.0 # Programmatically identical

3 < π

1 <= 1

.1 + .2

.1 + .2 ≈ .3 # \approx + TAB

# ------------------------------------------------------------------------------------------
# Comparisons "chain"
#
# Try inserting parentheses around one of these comparisons
# ------------------------------------------------------------------------------------------

2 == 2.0 == 0x02

x = 42
0 < x < 100 || error("x must be between 0 and 100")

# ------------------------------------------------------------------------------------------
# # Assignment
#
# Assignment in Julia is done with the single `=`. All it does is associates a name (on the
# left) to a value (on the right).
# ------------------------------------------------------------------------------------------

x = 1 # Use the name `x` for the value `1`

y = x # Use the name `y` for the value that `x` refers to

x = "hello!" # Decide you have a better use for the name `x`

y # Is still the value 1

ϵ = eps(1.0) # You can make your own unicode names

5ϵ # Juxtaposition is multiplication

# ------------------------------------------------------------------------------------------
# ## Updating operators
#
# All the infix arithmetic operators above can be used as "updating" operators in
# conjunction with an assignment:
# ------------------------------------------------------------------------------------------

y += 1

# This is exactly the same as:
y = y + 1

# ------------------------------------------------------------------------------------------
# Note that it's just re-purposing the _same name_ for a new value. This means that the type
# might even change:
# ------------------------------------------------------------------------------------------

y /= 2

# ------------------------------------------------------------------------------------------
# # Strings
# ------------------------------------------------------------------------------------------

s1 = "I am a string."

s2 = """I am also a string. """

"Here, we get an "error" because it's ambiguous where this string ends "

"""Look, Mom, no "errors"!!! """

    println("""The other nice thing about triple-quoted
               string literals is that they ignore leading
               indentation, which is nice for long strings
               in real code. Try changing these quotes!""")

# ------------------------------------------------------------------------------------------
# Strings are not written with single `'`s — that's used for a single character:
# ------------------------------------------------------------------------------------------

first(s1)

'⊂'

'If you try writing a string in single-quotes, you will get an error'

# ------------------------------------------------------------------------------------------
# ## String interpolation
# ------------------------------------------------------------------------------------------

# ------------------------------------------------------------------------------------------
# You can use the dollar sign inside a string to evaluate a Julia expression inside a string
# — either a single variable or a more complicated expression:
# ------------------------------------------------------------------------------------------

name = "Jane"
num_fingers = 10
num_toes = 10
println("Hello, my name is $name.")
println("I have $num_fingers fingers and $num_toes toes.")

 println("That is $(num_fingers + num_toes) digits in all!!")

@assert days == 365
@assert days_float == 365.0

.nbexports/introductory-tutorials/intro-to-julia/long-version/010 Arithmetic, Assignment, and Strings.jl


export Strang
"""
`Strang` matrix

A special `SymTridiagonal` matrix named after Gilbert Strang

```julia
julia> Strang(6)
6x6 Strang{Float64}:
  2.0  -1.0   0.0   0.0   0.0   0.0
 -1.0   2.0  -1.0   0.0   0.0   0.0
  0.0  -1.0   2.0  -1.0   0.0   0.0
  0.0   0.0  -1.0   2.0  -1.0   0.0
  0.0   0.0   0.0  -1.0   2.0  -1.0
  0.0   0.0   0.0   0.0  -1.0   2.0
```
"""
struct Strang{T} <: AbstractArray{T, 2}
    n :: Int
end
Strang(n::Int) = Strang{Float64}(n)
strang(T, n)= n >1 ? SymTridiagonal(2ones(T, n),-ones(T, n-1)) :
              n==1 ? Diagonal([2one(T)]) : error("Invalid dimension ", n)
function getindex(S::Strang{T}, i, j) where T
    i == j && return 2
    abs(i - j) == 1 && return -1
    0
end
getindex(S::Strang{T}, I...) where T = getindex(S,ind2sub(size(S),I...)...)
size(S::Strang, r::Int) = r==1 || r==2 ? S.n : throw(ArgumentError("Invalid dimension $r"))
size(S::Strang) = S.n, S.n
Matrix(S::Strang{T}) where T = Matrix(strang(T, S.n))
*(A::VecOrMat{T}, S::Strang{T}) where T = A*Matrix(S)
*(S::Strang{T}, A::VecOrMat{T}) where T = Matrix(S)*A


# Summary statistics

"""
    genvar(X)

Compute the generalized sample variance of `X`. If `X` is a vector, one-column matrix,
or other one-dimensional iterable, this is equivalent to the sample variance.
Otherwise if `X` is a matrix, this is equivalent to the determinant of the covariance
matrix of `X`.

!!! note
    The generalized sample variance will be 0 if the columns of the matrix of deviations
    are linearly dependent.
"""
genvar(X::AbstractMatrix) = size(X, 2) == 1 ? var(vec(X)) : det(cov(X))
genvar(itr) = var(itr)

"""
    totalvar(X)

Compute the total sample variance of `X`. If `X` is a vector, one-column matrix,
or other one-dimensional iterable, this is equivalent to the sample variance.
Otherwise if `X` is a matrix, this is equivalent to the sum of the diagonal elements
of the covariance matrix of `X`.
"""
totalvar(X::AbstractMatrix) = sum(var(X, 1))
totalvar(itr) = var(itr)


module EllipsoidAlgebra

using Plots, LinearAlgebra

"""
E(x, P) = { z : (z-x) P ( z-x) <= 1
"""
struct Ellipsoid{Tx, TP}
    x0::Tx
    P::TP
end

Ellipsoid(x0, P, V) = Ellipsoid(x0, P/V)

function plotEllipse!(e::Ellipsoid; kwargs...)

    @assert length(e.x0) == 2

    Pinv_half = real.(sqrt(inv(e.P)))

    x(θ) = e.x0[1] + first(Pinv_half * [cos(θ), sin(θ)])
    y(θ) = e.x0[2] + last(Pinv_half * [cos(θ), sin(θ)])

    plot!(x, y, 0, 2π; kwargs...)

end

function project(e::Ellipsoid, dims=[1,2])

    N = length(e.x0)
    Pinv_half = sqrt(inv(e.P))

    # construct projection matrix
    T = zeros(2, N)
    T[1,dims[1]] = 1
    T[2, dims[2]] = 1

    # get new shape matrix
    TPinvhalf = T * Pinv_half
    PPinv = TPinvhalf * TPinvhalf'
    Pp = real.(inv(PPinv))

    # get new center
    xp0 = T * e.x0

    return Ellipsoid(xp0, Pp)

end

function sampleRand(e::Ellipsoid, N=1)

    @assert isposdef(e.P)

    Pinv_half = sqrt(inv(e.P))

    # sample random from unit sphere, and map to ellipsoid
     y = 2*rand(length(e.x0), N) .- 1
     for i=1:N
        while norm(y[:,i]) > 1.0
            y[:,i] = 2*rand(length(e.x0), 1) .- 1
        end

    end

    xs = similar(y)
    for i=1:N
        xs[:,i] = e.x0 + Pinv_half * y[:,i]
    end

    return xs
end

function creategrid(d::Integer, n::Integer)

    @assert d >= 1 ("d (number of dimensions) must be a positive integer")
    @assert n >= 2 ("n (number of points) must be a at least 2")

    r = range(-1, 1, length = n)

    iter = Iterators.product((r for _ in 1:d)...)

    return vec([collect(i) for i in iter if dot(i, i) <= 1])
end

const sphere_6_grid = creategrid(6, 5)

function sampleUniformGrid(e::Ellipsoid, n=5)
    N = length(e.x0)
    Pinv_half = sqrt(inv(e.P))
    if N == 6
        pts = [e.x0 + Pinv_half * i for i in sphere_6_grid]
        return pts
    else
        pts = [e.x0 + Pinv_half * i for i in creategrid(N, n)]
        return pts
    end
end

end


using BenchmarkTools

# Get the number of iterations from the command line
n, = size(ARGS)
if n < 1
    println("Usage: TestGaussLegendreQuadrature.jl N")
    println("       ---> Please specify the number of grid points.")
    exit()
end

N = parse(Int, ARGS[1])

"""
    gauss(a, b, N)

Finds the approximate integral over the region [a, b]
with N iterations.
"""
function gauss(a, b, N)
    lambda, Q = eig(SymTridiagonal(zeros(N), [n/sqrt(4n^2 - 1) for n = 1:N-1]))
    return (lambda+1)*(b-a)/2 + a, [2*Q[1, i]^2 for i = 1:N]*(b-a)/2
end

println("--------------------------")
println(@sprintf "Gauss-Legendre Quadrature %d" N)
println("--------------------------")

@btime begin
   x, w = gauss(-3, 3, $N)
   quad = sum(exp.(x) .* w)
end

#exact = exp(3) - exp(-3)
#println(quad)
#println(exact)


@doc raw"""
    MaternKernel([ν=1 [,θ=1]])

The Matern kernel is a Mercer kernel given by:

```math
\kappa(\mathbf{x},\mathbf{y}) =
\frac{1}{2^{\nu-1}\Gamma(\nu)}
\left(\frac{2\sqrt{\nu}||\mathbf{x}-\mathbf{y}||}{\theta}\right)^{\nu}
K_{\nu}\left(\frac{2\sqrt{\nu}||\mathbf{x}-\mathbf{y}||}{\theta}\right)
```

where ``\Gamma`` is the gamma function, ``K_{\nu}`` is the modified Bessel function of the
second kind, ``\nu > 0`` and ``\theta > 0``.
"""
struct MaternKernel{T<:AbstractFloat} <: MercerKernel{T}
    nu::HyperParameter{T}
    rho::HyperParameter{T}
    MaternKernel{T}(ν::Real, ρ::Real) where {T<:AbstractFloat}  = new{T}(
        HyperParameter(convert(T,ν), interval(OpenBound(zero(T)), nothing)),
        HyperParameter(convert(T,ρ), interval(OpenBound(zero(T)), nothing))
    )
end
function MaternKernel(ν::T1=1.0, ρ::T2=one(T1)) where {T1<:Real,T2<:Real}
    MaternKernel{floattype(T1,T2)}(ν,ρ)
end

@inline function maternkernel(z::T, ν::T, ρ::T) where {T}
    v1 = sqrt(2ν) * z / ρ
    v1 = v1 < eps(T) ? eps(T) : v1  # Overflow risk as z -> Inf
    2 * (v1/2)^(ν) * besselk(ν, v1) / gamma(ν)
end

@inline basefunction(::MaternKernel) = SquaredEuclidean()
@inline function kappa(κ::MaternKernel{T}, z::T) where {T}
    maternkernel(z, getvalue(κ.nu), getvalue(κ.rho))
end


using PyPlot
using Optim

function get_ln_delta(N, d_VC, epsilon)
    return log(4) + d_VC * log(2N) -(epsilon^2)*N/8
end;

d_VC = 10;
epsilon = 0.05;
N_arr = [400000. 420000. 440000. 460000. 480000.];
for N in N_arr
    delta = exp(get_ln_delta(N, d_VC, epsilon));
    println("N = $N, delta = $delta")
end

function get_error(N, d_VC, epsilon, target_delta)
    delta = exp(get_ln_delta(N, d_VC, epsilon));
    return (delta - target_delta)^2;
end;

result = optimize(N -> get_error(N, 10, 0.05, 0.05), 400000, 500000)

N = result.minimum;
err = result.f_minimum;
if result.converged == true
    println("Solution:  N = $N  (error = $err)")
end

function get_vc_bound(N,d_VC,delta)
    inner = log(4) + d_VC * log(2N) - log(delta);
    return sqrt(8inner/N);
end;

function get_rademacher_penalty(N,d_VC,delta)
    inner = 2(log(2N) + d_VC * log(N))/N
    return sqrt(inner) + sqrt(-2log(delta)/N) + 1/N;
end;

function get_parrondo_and_van_den_broek(N,d_VC,delta)
    pvdb_err(epsilon,N,d_VC,delta) = (epsilon - sqrt((2epsilon + log(6) + d_VC*log(2N) - log(delta))/N))^2;
    result = optimize(epsilon -> pvdb_err(epsilon,N,d_VC,delta), 0.0, 10.0);
    if result.converged==true
        return result.minimum
    else
        return NaN
    end;
end;

function get_devroye(N,d_VC,delta)
    devroye_err(epsilon,N,d_VC,delta) = (epsilon - sqrt((4epsilon*(1+epsilon) + log(4) + d_VC*log(N^2) - log(delta))/(2N)))^2;
    result = optimize(epsilon -> devroye_err(epsilon,N,d_VC,delta), 0.0, 20.0);
    if result.converged==true
        return result.minimum
    else
        return NaN
    end;
end;

d_VC = 50;
delta = 0.05;

#N_arr = logspace(0,4,11);  #from 1 to 10k
N_arr = [5, 10, 50, 100, 500, 1000, 5000, 10000];
epsilon_vc = [ get_vc_bound(N,d_VC,delta) for N in N_arr ];
epsilon_rp = [ get_rademacher_penalty(N,d_VC,delta) for N in N_arr ];
epsilon_pv = [ get_parrondo_and_van_den_broek(N,d_VC,delta) for N in N_arr ];
epsilon_dv = [ get_devroye(N,d_VC,delta) for N in N_arr ];

function plot_epsilons(N_arr,epsilon_arr,label_arr)
    fig = figure(figsize=(7,4))
    ax = fig[:add_subplot](1,1,1)
    for i in 1:size(label_arr)[1]
        #ax[:loglog](N_arr,epsilon_arr[:,i],label=label_arr[i]);
        loglog(N_arr,epsilon_arr[:,i],"x-",label=label_arr[i]);
    end;
    #ax[:legend](loc="best",frameon=false);
    legend(loc="best",frameon=false,fontsize=10)
    grid(true);
end;

epsilon_arr = [ epsilon_vc epsilon_rp epsilon_pv epsilon_dv ];
label_arr = [ "VC", "Rademacher", "Parrondo & Van den Broek", "Devroye" ];
plot_epsilons(N_arr,epsilon_arr,label_arr);

println("N = $(N_arr[end])")
println("ϵ (VC)                       = $(epsilon_vc[end])")
println("ϵ (Rademacher)               = $(epsilon_rp[end])")
println("ϵ (Parrondo & Van den Broek) = $(epsilon_pv[end])")
println("ϵ (Devroye)                  = $(epsilon_dv[end])")

println("N = $(N_arr[1])")
println("ϵ (VC)                       = $(epsilon_vc[1])")
println("ϵ (Rademacher)               = $(epsilon_rp[1])")
println("ϵ (Parrondo & Van den Broek) = $(epsilon_pv[1])")
println("ϵ (Devroye)                  = $(epsilon_dv[1])")

function generate_dataset(;n_points=200)
    x = linspace(-1,1,n_points);
    y = sin(π*x);
    x_sample = rand(2);
    y_sample = sin(π*x_sample);
    return x,y,x_sample,y_sample;
end;

function fit_a(x_sample,y_sample)
    sum = x_sample[1]*y_sample[1]+x_sample[2]*y_sample[2]
    return sum/(x_sample[1]^2+x_sample[2]^2);
end;

function predict_y_a(x,a)
    return a .* x;
end;

function plot_xy(x,y,x_sample,y_sample,y_pred)
    figure(figsize=(7,5));
    plot(x,y,"m-",label=L"$\sin(\pi x)$");
    plot(x,y_pred,"g--",label="predicted");
    plot(x_sample,y_sample,"bo",label="sample data");
    grid(true);
    legend(loc="best",frameon=false);
end;
x,y,x_sample,y_sample = generate_dataset();
y_pred = predict_y_a(x,fit_a(x_sample,y_sample));
plot_xy(x,y,x_sample,y_sample,y_pred);

function get_avg_a(n_trials; n_points=200)
    a_arr = Float64[];
    for i in 1:n_trials
        x_sample = rand(2);
        y_sample = sin(π*x_sample);
        a = fit_a(x_sample,y_sample);
        push!(a_arr,a);
    end
    return mean(a_arr);
end;
a = get_avg_a(1e6);    #1.423, 1.425, 1.424, 1.427
println("<g(x)> = $(round(a,2))x");

function get_bias_variance(avg_a, n_trials; n_points=200)
    bias  = Float64[];
    var   = Float64[];
    for i in 1:n_trials
        x_sample = rand(2);
        y_sample = sin(π*x_sample);
        a = fit_a(x_sample,y_sample);
        g_bar_x = avg_a*x_sample;
        push!(bias,mean((g_bar_x - y_sample).^2));
        push!(var,mean((g_bar_x - a .* x_sample).^2));
    end
    return bias, var;
end;

bias,var = get_bias_variance(a, 1e7);
println("<bias> = $(mean(bias))");
println("<var>  = $(mean(var))");


"""
    GaussianCopula{d,MT}

Fields:
  - Σ::MT - covariance matrix

Constructor

    GaussianCopula(Σ)

The [Gaussian Copula](https://en.wikipedia.org/wiki/Copula_(probability_theory)#Gaussian_copula) is the
copula of a [Multivariate normal distribution](http://en.wikipedia.org/wiki/Multivariate_normal_distribution). It is constructed as :

```math
C(\\mathbf{x}; \\boldsymbol{\\Sigma}) = F_{\\Sigma}(F_{\\Sigma,i}^{-1}(x_i),i\\in 1,...d)
```
where ``F_{\\Sigma}`` is a cdf of a gaussina random vector and `F_{\\Sigma,i}` is the ith marignal cdf, while ```\\Sigma`` is the covariance matrix.

It can be constructed in Julia via:
```julia
C = GaussianCopula(Σ)
```

The random number generation works as expected:
```julia
rand(C,1000)
# or
Random.rand!(C,u)
```

And yo can fit the distribution via :
```julia
fit(GaussianCopula,data)
```
"""
struct GaussianCopula{d,MT} <: EllipticalCopula{d,MT}
    Σ::MT
    function GaussianCopula(Σ)
        make_cor!(Σ)
        return new{size(Σ,1),typeof(Σ)}(Σ)
    end
end
U(::Type{T}) where T<: GaussianCopula = Distributions.Normal
N(::Type{T}) where T<: GaussianCopula = Distributions.MvNormal
function Distributions.fit(::Type{CT},u) where {CT<:GaussianCopula}
    dd = Distributions.fit(N(CT), quantile.(U(CT)(),u))
    Σ = Matrix(dd.Σ)
    return GaussianCopula(Σ)
end


function bisection(f′, a, b, ϵ)
    if a > b; a, b = b, a; end
    ya, yb = f′(a), f′(b)
    if ya == 0; b = a; end
    if yb == 0; a = b; end
    while b-a > ϵ
        x = (a+b)/2;
        y = f′(x);
        if y == 0
            a, b = x, x;
        elseif sign(y) == sign(ya)
            a = x;
        else
            b = x;
        end
    end
    return (a, b);
end


function blkdiag(blocks::Array{Array{Pol{T}, 2}, 1}) where {T<:Number}
    m = sum(size.(blocks, 1))
    n = sum(size.(blocks, 2))
    J = zeros(Pol{T}, m, n)
    offset1 = 0
    offset2 = 0
    for blk in blocks
        J[offset1+1:offset1+size(blk, 1), offset2+1:offset2+size(blk, 2)] = blk
        offset1 += size(blk, 1)
        offset2 += size(blk, 2)
    end
    return J
end

"""
creates the matrix block of size n×(n+1) corresponding to column index n
"""
function colBlock(n::Int)
    var = Pol([0, 1])
    J = zeros(Pol{Int}, n, n+1)
    for ii in 1:n
        J[ii, ii] = var
        J[ii, ii+1] = -1
    end
    return J
end

colBlock(arr::AbstractArray{Int, 1}) = blkdiag(colBlock.(arr))

"""
creates the matrix block of size (n+1)×n corresponding to row index n
"""
function rowBlock(n::Int)
    var = Pol([0, 1])
    J = zeros(Pol{Int}, n+1, n)
    @inbounds for ii in 1:n
        J[ii, ii] = -1
        J[ii+1, ii] = var
    end
    return J
end

rowBlock(arr::AbstractArray{Int, 1}) = blkdiag(rowBlock.(arr))

"""
creates the block matrix corresponding to infinite eigenvalue of multiplicity n
"""
function infBlock(n::Int)
    var = Pol([0, 1])
    J = zeros(Pol{Int}, n, n)
    @inbounds for ii in 1:n-1
        J[ii, ii] = -1
        J[ii+1, ii] = var
    end
    J[n, n] = -1
    return J
end

infBlock(arr::AbstractArray{Int, 1}) = blkdiag(infBlock.(arr))

"""
creates the block matrix of size n×n corresponding to eigenvalue λ
"""
function eigBlock(λ::T, n::Int) where {T<:Number}
    var = Pol([0, 1])
    J = zeros(Pol{T}, n, n)
    @inbounds for ii in 1:n-1
        J[ii, ii] = var - λ
        J[ii+1, ii] = -1
    end
    J[n, n] = var - λ
    return J
end

eigBlock(λ::Number, arr::AbstractArray{Int, 1}) = blkdiag(eigBlock.(λ, arr))


"""
    SEIso <: Isotropic{SqEuclidean}

Isotropic Squared Exponential kernel (covariance)
```math
k(x,x') = σ²\\exp(- (x - x')ᵀ(x - x')/(2ℓ²))
```
with length scale ``ℓ`` and signal standard deviation ``σ``.
"""
mutable struct SEIso{T<:Real} <: Isotropic{SqEuclidean}
    "Squared length scale"
    ℓ2::T
    "Signal variance"
    σ2::T
    "Priors for kernel parameters"
    priors::Array
end

"""
Squared Exponential kernel function

    SEIso(ll::T, lσ::T)

# Arguments:
- `ll::Real`: length scale (given on log scale)
- `lσ::Real`: signal standard deviation (given on log scale)
"""
SEIso(ll::T, lσ::T) where T = SEIso{T}(exp(2 * ll), exp(2 * lσ), [])

function set_params!(se::SEIso, hyp::AbstractVector)
    length(hyp) == 2 || throw(ArgumentError("Squared exponential has two parameters, received $(length(hyp))."))
    se.ℓ2, se.σ2 = exp(2 * hyp[1]), exp(2 * hyp[2])
end

get_params(se::SEIso{T}) where T = T[log(se.ℓ2) / 2, log(se.σ2) / 2]
get_param_names(se::SEIso) = [:ll, :lσ]
num_params(se::SEIso) = 2

cov(se::SEIso, r::Number) = se.σ2*exp(-0.5*r/se.ℓ2)

@inline dk_dll(se::SEIso, r::Real) = r/se.ℓ2*cov(se,r)
@inline function dk_dθp(se::SEIso, r::Real, p::Int)
    if p==1
        return dk_dll(se, r)
    elseif p==2
        return dk_dlσ(se, r)
    else
        return NaN
    end
end


export frac_iso, is_frac_iso

"""
    frac_iso(G,H)

Find a doubly stochastic matrix `S` so that `AS=SB` where `A` and `B`
are the adjacency matrices of `G` and `H` respectively.
"""
function frac_iso(G::SimpleGraph, H::SimpleGraph)
    if deg(G) != deg(H) || degdeg(G) != degdeg(H) # quick basic check before LP
        error("The graphs are not fractionally isomorphic")
    end
    A = adjacency(G)
    B = adjacency(H)
    return frac_iso(A, B)
end

function frac_iso(A::Matrix, B::Matrix)
    n, c = size(A)
    nn, cc = size(B)
    @assert n == c && nn == cc "Matrices must be square"
    @assert n == nn "Matrices must have the same size"

    m = Model(get_solver())

    @variable(m, S[1:n, 1:n])

    for i = 1:n
        for k = 1:n
            @constraint(
                m,
                sum(A[i, j] * S[j, k] for j = 1:n) == sum(S[i, j] * B[j, k] for j = 1:n)
            )
        end
        @constraint(m, sum(S[i, j] for j = 1:n) == 1)
        @constraint(m, sum(S[j, i] for j = 1:n) == 1)
    end

    for i = 1:n
        for j = 1:n
            @constraint(m, S[i, j] >= 0)
        end
    end

    optimize!(m)
    status = Int(termination_status(m))

    if status != 1
        error("The graphs are not fractionally isomorphic")
    end

    return value.(S)

end

"""
    is_frac_iso(G,H)
Test if two graphs are fractionally isomorphic.
"""
function is_frac_iso(G::SimpleGraph, H::SimpleGraph)::Bool
    try
        f = frac_iso(G, H)
        return true
    catch
        return false
    end
    false
end


using EngEconomics, Roots, Plots, ColorSchemes
plotly()

# Given
constructionYears = 5 # Construction takes this many years
constructionStarts = 5 # Construction starts in this many years
constructionA = 20000 # Construction annual cost in $1000, so if this number is 20,000, the real price is 20,000,000
maintenanceStarts = 1 + constructionYears + constructionStarts # Number of years when maintenance for the project will begin. It is simply the sum of how long construction will take and when construction starts plus 1 year
maintenanceA = 2000 # Maintenance annual cost in $1000, so if this number is 2,000, then the real price is 2,000,000
inflationRate = 0.03 # Inflation rate
MARR = 0.07 # Project's MARR
projectLifeTime = 30 # I did this in my head (starts working in 2020, supposed to last till 2050)

# Find
# (a.) The present worth of the project
realMARR = (1 + MARR) / (1 + inflationRate) - 1
NPW = (-constructionA * seriesPresentAmountFactor(realMARR, constructionYears)) * presentWorthFactor(realMARR, constructionStarts) + (-maintenanceA * seriesPresentAmountFactor(realMARR, projectLifeTime)) * presentWorthFactor(realMARR, constructionYears + constructionStarts)

# (b.) Construct a sensitivity graph showing the effects of 5 percent and 10
#      percent increases and decreases in the construction costs, maintenance
#      costs and inflation rate. To which is the present worth most sensitive?
sensitivity = [0.9, 0.95, 1.0, 1.05, 1.1]

# Construction Costs
realMARR = (1 + MARR) / (1 + inflationRate) - 1
NPW_constructionCost = (-constructionA * sensitivity * seriesPresentAmountFactor(realMARR, constructionYears)) * presentWorthFactor(realMARR, constructionStarts) .+ (-maintenanceA * seriesPresentAmountFactor(realMARR, projectLifeTime)) * presentWorthFactor(realMARR, constructionYears + constructionStarts)
plot(sensitivity, NPW_constructionCost)

# Maintenance Costs
realMARR = (1 + MARR) / (1 + inflationRate) - 1
NPW_maintenanceCost = (-constructionA * seriesPresentAmountFactor(realMARR, constructionYears)) * presentWorthFactor(realMARR, constructionStarts) .+ (-maintenanceA * sensitivity * seriesPresentAmountFactor(realMARR, projectLifeTime)) * presentWorthFactor(realMARR, constructionYears + constructionStarts)
plot!(sensitivity, NPW_maintenanceCost)

# Inflation Rate
realMARR_inflationRate = (1 + MARR) ./ (1 .+ sensitivity * inflationRate) .- 1
NPW_inflationRate = (-constructionA * seriesPresentAmountFactor.(realMARR_inflationRate, constructionYears)) .* presentWorthFactor.(realMARR_inflationRate, constructionStarts) .+ (-maintenanceA * seriesPresentAmountFactor.(realMARR_inflationRate, projectLifeTime)) .* presentWorthFactor.(realMARR_inflationRate, constructionYears + constructionStarts)
plot!(sensitivity, NPW_inflationRate)

println("From the plot, it\'s clear that the NPW cost is most sensitive to the initial construction cost because the slope of that line is much bigger than that of the other lines")


#=
Please see
https://computationalmindset.com/en/neural-networks/ordinary-differential-equation-solvers.html#ode1
for details
=#

#=Original problem
x′ + x = sin t + 3 cos 2t
x(0) = 0
=#

#=Esplicit form of equation
x′ = sin t  + 3 cos 2t − x
=#

#=Analytical solution
x(t) = 1/2 sin t - 1/2 cos t + 3/5 cos 2t + 6/5 sin 2t - 1/10 e^-t
=#

using DifferentialEquations
using Plots

ode_fn(x,p,t) = sin(t) + 3.0 * cos(2.0 * t) - x

an_sol(t) = (1.0/2.0) * sin(t) - (1.0/2.0) * cos(t) +
            (3.0/5.0) * cos(2.0*t) + (6.0/5.0) * sin(2.0*t) -
            (1.0/10.0) * exp(-t)

t_begin=0.0
t_end=10.0
tspan = (t_begin,t_end)
x_init=0.0

prob = ODEProblem(ode_fn, x_init, tspan)
num_sol = solve(prob, Tsit5(), reltol=1e-8, abstol=1e-8)

plot(num_sol.t, an_sol.(num_sol.t),
    linewidth=2, ls=:dash,
    title="ODE 1st order IVP solved by D.E. package",
    xaxis="t", yaxis="x",
    label="analytical",
    legend=true)
plot!(num_sol,
    linewidth=1,
    label="numerical")


"""
    PInterval <: SimpleProbabilitySet

Credal set of categorical distributions defined by lower and upper probabilities.
"""
struct PInterval <: SimpleProbabilitySet
    plower::Vector{Float64}
    pupper::Vector{Float64}
end

"""
    PInterval(p::Vector{Float64}, uncsize::Float64)

Probability intervals centered on 'p' of width ± 'uncsize' (with saturated bounds)
"""
function PInterval(p::Vector{Float64}, uncsize::Float64)
    pl = zeros(length(p))
    pu = ones(length(p))
    for (i, pi) in enumerate(p)
        pl[i] = max(0.0, pi - uncsize)
        pu[i] = min(1.0, pi + uncsize)
    end
    PInterval(pl, pu)
end

"""
    PInterval(p::Vector{Float64})

Degenerate probability interval of only pmf 'p'
"""
PInterval(p::Vector{Float64}) = PInterval(p,p) # degenerate PInterval

"""
    PInterval()

Default probability interval of p1 ∈ [0.2, 0.3], p2 ∈ [0.7, 0.8]
"""
PInterval() = PInterval([0.2, 0.7], [0.3, 0.8]) # default PInterval

"""
    psample(plower::Vector{Float64}, pupper::Vector{Float64})

Generate a probability distribution, uniformly at random, from the ambiguity set defined by the `pupper` and `plower` probabilities.
"""
function psample(plower::Vector{Float64}, pupper::Vector{Float64})
    n = length(plower)
    A = zeros(Float64, 2n+1, n)
    for i = 1:n
        A[2i - 1,i] = 1.0
        A[2i, i] = -1.0
    end
    A[end,:] = ones(n)
    bc = Array{Float64}(2n+1)
    for i = 1:n
        # due to limitations in hitandrun, set upper limit to 0+eps if upper limit is actually 0.0
        bc[2i - 1] = pupper[i] == 0.0 ? pupper[i] + 0.0000001 : pupper[i]
        bc[2i] = -1.0 * plower[i]
    end
    bc[end] = 1.0
    d = vcat(fill("<=", 2n),"=")
    nsample = 1
    rseed = rand(1:10_000)
    rout = R"""
    library(hitandrun)
    set.seed($rseed)
    constr <- list(constr = $A, rhs = $bc, dir = $d)
    samples <- hitandrun(constr, n.samples = $nsample, thin = ($n) ^ 3)
    """
    p = reshape(rcopy(rout), n)
end

psample(pint::PInterval) = psample(pint.plower, pint.pupper)


module problem126

export Cuboid

description = """
Cuboid layers

The minimum number of cubes to cover every visible face on a cuboid measuring 3 x 2 x 1 is twenty-two.
If we then add a second layer to this solid it would require forty-six cubes to cover every visible face,
the third layer would require seventy-eight cubes, and the fourth layer would require one-hundred and eighteen cubes to cover every visible face.

However, the first layer on a cuboid measuring 5 x 1 x 1 also requires twenty-two cubes;
similarly the first layer on cuboids measuring 5 x 3 x 1, 7 x 2 x 1, and 11 x 1 x 1 all contain forty-six cubes.

We shall define C(n) to represent the number of cuboids that contain n cubes in one of its layers.
So C(22) = 2, C(46) = 4, C(78) = 5, and C(118) = 8.
It turns out that 154 is the least value of n for which C(n) = 10.

Find the least value of n for which C(n) = 1000.
"""

# A cube with length, width and height
immutable Cuboid
  l :: Int
  w :: Int
  h :: Int
end

# calculates the middle section of the n'th layer
# equal to (the perimeter of original cube + diagonals) * height or original cube
middle(c :: Cuboid, n :: Int) = (2 * (c.l + c.w) + 4 * (n - 1)) * c.h

# Recurrance relation for calculating the top section of the n'th layer
# For n = 1, just the area of the top of original cube
# otherwise, equal to the top of the previous layer + a one cube thick rim from the middle of previous layer
top(c :: Cuboid, n :: Int) =
  if (n == 1)
    return (c.l * c.w)
  else
    return (top(c, n-1) + div(middle(c, n-1), c.h))
  end

# Calculate the n'th layer, top and bottom are symetric + middle section
layer(c :: Cuboid, n :: Int) = 2 * top(c, n) + middle(c, n)

# Calculate the size of all layers of cuboid up to a maximum
# re-uses the results of previous iterations to efficiently calculate the next
function layers(c :: Cuboid, maxsize :: Int)
  n = 1
  t = top(c, n)
  m = middle(c, n)
  x = 0
  result = Int[]
  while true
    x = 2 * t + m
    if x > maxsize break end
    push!(result, x)
    t = t + div(m, c.h)
    n += 1
    m = middle(c, n)
  end
  return result
end

# Search for a solution x, to C(x) == N, for x in 1:guess
# The lower bound is given by layer 1 of cuboid (Length, 1, 1)
# Once a solution is found < lower bound, search can stop
function search(N :: Int, guess :: Int)
  counts = zeros(Int, guess)
  l = 1
  lowerbound = layer(Cuboid(l, 1, 1), 1)
  while lowerbound <= guess
    for i = 1:lowerbound-1
      if counts[i] == N
        return i
      end
    end
    w = 1
    while w <= l && layer(Cuboid(l, w, 1), 1) <= guess
      h = 1
      while h <= w && layer(Cuboid(l, w, h), 1) <= guess
        sizes = layers(Cuboid(l,w,h), guess)
        counts[sizes] += 1
        h += 1
      end
      w += 1
    end
    l += 1
    lowerbound = layer(Cuboid(l, 1, 1), 1)
  end
  println("no solution found < $guess")
  return -1
end

using Base.Test

@test layer(Cuboid(3, 2, 1), 1) == 22
@test layer(Cuboid(3, 2, 1), 2) == 46
@test layer(Cuboid(3, 2, 1), 3) == 78
@test layer(Cuboid(3, 2, 1), 4) == 118
@test layer(Cuboid(5, 1, 1), 1) == 22
@test layer(Cuboid(5, 3, 1), 1) == 46
@test layer(Cuboid(7, 2, 1), 1) == 46
@test layer(Cuboid(11, 1, 1), 1) == 46
@test layers(Cuboid(3,2,1), 78) == [22, 46, 78]
@test search(10, 200) == 154

end


function atan2(y,x)
    if x>0
        return atan(y/x)
    elseif (x<0 && y>=0)
        return atan(y/x)+pi
    elseif (x<0 && y<0)
        return atan(y/x)-pi
    elseif (x==0 && y>0)
        return pi/2
    elseif (x==0 && y<0)
        return -pi/2
    else
        return NaN
    end
end


# Fixed point classification in 2D
using DrWatson
@quickactivate "NonlinearDynamicsTextbook"
include(srcdir("style.jl"))
using DynamicalSystems, PyPlot, Random

alleigs = []
fig, axs = subplots(2,3; figsize = (figx, 2figy))
ax = gca()
xgrid = -5:0.05:5
ygrid = xgrid
ux = zeros(length(xgrid), length(ygrid))
uy = copy(ux)

Ms = [
     [1.0  0.8;
      0.4 1.0],
     [-1.0  -0.8;
      1.8  -1.0],
     [-1.0  0;
      1  1],
     [0 -1;
      1  0],
]

titles = ["repulsive node", "attractive spiral", "hyperbolic/saddle", "center"]

using LinearAlgebra
function stream_eigs!(ax, M, c = "C0")
    for (i, x) in enumerate(xgrid)
        for (j, y) in enumerate(ygrid)
            ux[i, j], uy[i, j] = M * [x, y]
        end
    end

    ax.streamplot(Vector(xgrid), Vector(ygrid), ux', uy';
        linewidth = 1.5, density = 0.5, color = c, arrowsize = 2,
    )
    ev = eigen(M)
    push!(alleigs, ev.values)
    if eltype(ev.values) <: Float64
        e1 = ev.vectors[:, 1]
        e2 = ev.vectors[:, 2]
        for e in (e1, e2)
            ax.plot(e[1] .* 2xgrid, e[2] .* 2ygrid; color = "C2", ls = "dashed")
        end
    end
    ax.set_xlim(xgrid[1], xgrid[end])
    ax.set_ylim(ygrid[1], ygrid[end])
    ax.set_xticks([])
    ax.set_yticks([])
    ax.plot(0, 0; marker = "o", mec = "C2", mew = 1,
        markersize = 12, mfc = :white, zorder = 99
    )
end

function tovec(e)
    if eltype(e) <: Real
        e1 = e
        e2 = zeros(2)
    else
        e1 = [e[1].re, e[2].re]
        e2 = [e[1].im, e[2].im]
    end
    return e1, e2
end

for (i, M) in enumerate(Ms)
    ax = axs[i]
    color = i < 3 ? "C$(i-1)" : "C$(i)"
    stream_eigs!(ax, M, color)
    ax.set_title(titles[i]; color = color)
    # Plot eigenvalues
    e1, e2 = tovec(alleigs[i])
    axs[5].scatter(e1, e2; color = color, s = 200, zorder = 99)
end

# Set axis of eigenval plot
axs[5].spines["left"].set_position("center")
axs[5].spines["bottom"].set_position("center")
axs[5].grid(false)
axs[5].spines["right"].set_color("none")
axs[5].spines["top"].set_color("none")
axs[5].set_title("eigenvalues"; color = "k")
# axs[5].set_yticklabels([])
# axs[5].set_xticklabels([])
axs[5].set_xlim(-2, 2)
axs[5].set_ylim(-1.8, 1.8)

# Plot limit cycle
function vanderpoll(u, p, t)
    x, y = u;
    xdot = p*(x - x^3/3 - y)
    ydot = x/p
    return SVector(xdot, ydot)
end

ds = ContinuousDynamicalSystem(vanderpoll, [0.1, 0.2], 0.5)
tr = trajectory(ds, 7.0; Ttr = 100.0)
axs[6].clear()
axs[6].plot(columns(tr)...; color = "C2", linestyle = "--")
for (i, x) in enumerate(xgrid)
    for (j, y) in enumerate(ygrid)
        ux[i, j], uy[i, j] = ds.f(SVector(x, y), ds.p, 0)
    end
end
axs[6].streamplot(Vector(xgrid), Vector(ygrid), ux', uy';
    linewidth = 1.5, density = 0.5, color = "C5", arrowsize = 2
)
ax = axs[6]
ax.set_xlim(xgrid[1], xgrid[end])
ax.set_ylim(ygrid[1], ygrid[end])
ax.set_xticks([])
ax.set_yticks([])
ax.set_title("attractive limit cycle"; color = "C5")

fig.subplots_adjust(bottom = 0.02, left = 0.02, top = 0.92, right = 0.97, hspace = 0.2)
wsave(plotsdir("2", "2ddynamics"), fig)


@testset "Test LogisticFunction"  begin
    L = LogisticFunction(3)

    x = [1, 2, 3]
    z = z_buffer(L)
    dx = dx_buffer(L)

    evaluate!(L, x, z, dx=dx)

    @test z ≈ [0.7310585786, 0.8807970779, 0.9525741268]
    @test dx ≈ [0.1966119332 0 0;
                0 0.1049935854 0;
                0 0 0.0451766597]
end

@testset "Test HyperbolicTangent"  begin
    H = HyperbolicTangent(3)

    x = [1, 2, 3]
    z = z_buffer(H)
    dx = dx_buffer(H)

    evaluate!(H, x, z, dx=dx)

    @test z ≈ [0.7615941559, 0.9640275801, 0.9950547537]
    @test dx ≈ [0.4199743416 0 0;
                0 0.0706508249 0;
                0 0 0.0098660372]
end

@testset "Test AffineMap"  begin
    @testset "Full Input"  begin
        A = AffineMap([2, 2, 4], [1, 2, 3])

        x = [1, 2, 3]
        z = z_buffer(A)
        dx = dx_buffer(A)

        evaluate!(A, x, z, dx=dx)

        @test z ≈ [3.0, 6.0, 15.0]
        @test dx ≈ [2.0 0 0;
                    0 2.0 0;
                    0 0 4.0]
    end
    @testset "Default Bias"  begin
        A = AffineMap([2, 2, 4])

        x = [1, 2, 3]
        z = z_buffer(A)
        dx = dx_buffer(A)

        evaluate!(A, x, z, dx=dx)

        @test z ≈ [2.0, 4.0, 12.0]
        @test dx ≈ [2.0 0 0;
                    0 2.0 0;
                    0 0 4.0]
    end

    @testset "Argument Error"  begin
        @test_throws ArgumentError AffineMap([2, 2, 4],
                                             [1, 1, 1, 1])
    end
end

@testset "Test Identity"  begin
    H = Identity(3)

    x = [1, 2, 3]
    z = z_buffer(H)
    dx = dx_buffer(H)

    evaluate!(H, x, z, dx=dx)

    @test z ≈ [1.0, 2.0, 3.0]
    @test dx ≈ [1.0 0 0;
                0 1.0 0;
                0 0 1.0]
end


@doc raw"""
    PowerKernel([γ=1])

The Power Kernel is a negative definite kernel given by:

```math
\kappa(\mathbf{x},\mathbf{y}) =
\|\mathbf{x} - \mathbf{y} \|^{2\gamma}
\qquad \gamma \in (0,1]
```
"""
struct PowerKernel{T<:AbstractFloat} <: NegativeDefiniteKernel{T}
    gamma::HyperParameter{T}
    PowerKernel{T}(γ::Real) where {T<:AbstractFloat} = new{T}(
        HyperParameter(convert(T,γ), interval(OpenBound(zero(T)), ClosedBound(one(T))))
    )
end
PowerKernel(γ::T1 = 1.0) where {T1<:Real} = PowerKernel{floattype(T1)}(γ)

@inline powerkernel(z::T, γ::T) where {T<:AbstractFloat} = z^γ

@inline basefunction(::PowerKernel) = SquaredEuclidean()
@inline kappa(κ::PowerKernel{T}, z::T) where {T} = powerkernel(z, getvalue(κ.gamma))


include("getdigit_n.jl")
include("myprintf.jl")
using Printf
using .GetDigit_n
using .MyPrintf

const DIGIT = 100
const MAXN = 500

function gaulegofpi(n)
    aₙ = BigFloat(1)
    bₙ = BigFloat(1) / sqrt(BigFloat(2))
    tₙ = parse(BigFloat, "0.25")
    pₙ = BigFloat(1)

    for i = 1:n
        aₙ₊₁ = (aₙ + bₙ) / BigFloat(2)
        bₙ = sqrt(aₙ * bₙ)
        tₙ = tₙ - pₙ * (aₙ - aₙ₊₁) * (aₙ - aₙ₊₁)
        pₙ = BigFloat(2) * pₙ
        aₙ = aₙ₊₁
    end

    return (aₙ + bₙ) * (aₙ + bₙ) / (BigFloat(4) * tₙ)
end

function main()
    n = GetDigit_n.getn(gaulegofpi, MAXN, DIGIT, true)
    precision = GetDigit_n.getdigit(gaulegofpi, n, DIGIT, true)
    precision = GetDigit_n.getdigit_2(gaulegofpi, n, DIGIT, precision)

    setprecision(precision)
    MyPrintf.myprintf("π（計算値） = ", DIGIT, gaulegofpi(n))
    MyPrintf.myprintf("π（厳密値） = ", DIGIT, BigFloat(π))
    @printf("n = %d, 計算%d桁で、%d桁求まりました\n", n, precision, DIGIT)
end

main()


"""
`assemble_system!` - assemble the matrix for the left hand side of the problem
"""
function assemble_system!(problem::RandomODEProblem)
    # construct the first row of the discrete Laplacian
    problem.A[1,1] = -2
    problem.A[1,2] = 1

    # construct middle rows
    for j ∈ 2:problem.nₓ - 1
        problem.A[j, j- 1] = 1
        problem.A[j, j] = -2
        problem.A[j, j + 1] = 1
    end

    # construct last row
    problem.A[problem.nₓ, problem.nₓ - 1] = 1
    problem.A[problem.nₓ, problem.nₓ] = -2

    # adjust for random potential
    problem.A = spdiagm(problem.V) - problem.A/problem.Δx^2

    return problem
end


using Pkg
Pkg.activate(".")
using PlotlyJS, Zygote, LinearAlgebra
using Dash, DashCoreComponents, DashHtmlComponents
using DashBootstrapComponents

function frenet_frame(x, y, z)
    ẋ(t) = gradient(x, t)[1]
    ẏ(t) = gradient(y, t)[1]
    ż(t) = gradient(z, t)[1]

    ẍ(t) = hessian(x, t)
    ÿ(t) = hessian(y, t)
    z̈(t) = hessian(z, t)

    α̇(t) = [ẋ(t), ẏ(t), ż(t)]
    α̈(t) = [ẍ(t), ÿ(t), z̈(t)]
    function out(t)
        T = α̇(t)
        T = T / norm(T)
        ad = α̈(t)
        N = ad - dot(ad, T)T
        N = N / norm(N)
        B = cross(T, N)
        T, N, B
    end
end

function uv_surface(uv_domain, ϕ; uv_res = [40, 40])
    us = LinRange(uv_domain[1], uv_domain[2], uv_res[1])
    vs = LinRange(uv_domain[3], uv_domain[4], uv_res[2])
    u_res, v_res = uv_res
    xs = zeros(u_res, v_res)
    ys = zeros(u_res, v_res)
    zs = zeros(u_res, v_res)
    for (i, u) in enumerate(us)
        for (j, v) in enumerate(vs)
            xs[i, j], ys[i, j], zs[i, j] = ϕ(u, v)
        end
    end
    surface(x = xs, y = ys, z = zs, showscale = false)
end

function xyz(ps)
    x = [p[1] for p in ps]
    y = [p[2] for p in ps]
    z = [p[3] for p in ps]
    return x, y, z
end

function torus_knot(m::Int64, n::Int64; R = 2, r = 1, uv_res = [220, 220])
    ϕ(u, v) = [(R + r * cos(u)) * cos(v), (R + r * cos(u)) * sin(v), r * sin(u)]
    α(t) = ϕ(m * t, n * t)
    x(t) = α(t)[1]
    y(t) = α(t)[2]
    z(t) = α(t)[3]

    F = frenet_frame(x, y, z)
    function ψ(u, v)
        T, N, B = F(u)
        ρ = r / 3
        α(u) + ρ * cos(v) * N + ρ * sin(v) * B
    end
    uv_surface([-π, 1.02π, -π, π], ψ, uv_res = uv_res)
end
description = dcc_markdown(raw"Torus knots are obtained by winding a curve around a torus
 ($m$ times in parallel direction and $n$ times in the meridian direction). More concretely, if

 $$\varphi(u,v) = ((R+r\cos(u))\cos(v), (R+r\cos(u))\sin(v), r\sin(u))$$

 is the torus parametrization, the curve

 $$\alpha(t) = \varphi(mt,nt)\text{ for }0\leq t \leq 2\pi $$

is said to be knot of the type $(m,n)$. These knots can be made into surfaces by thickening them using the Frenet frame over $\alpha$
 ")
sd = 4.0
scene3d = Dict(
    :xaxis => Dict(:range => [-sd, sd], :autorange => false),
    :yaxis => Dict(:range => [-sd, sd], :autorange => false),
    :zaxis => Dict(:range => [-sd, sd], :autorange => false),
    :aspectratio => Dict(:x => 1, :y => 1, :z => 1),
)

layout = Layout(scene = scene3d, width = 600, height = 600)

mathjax = "https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.4/MathJax.js?config=TeX-MML-AM_CHTML"
app = dash(external_scripts = [mathjax], external_stylesheets = [dbc_themes.BOOTSTRAP])
app.title = "Dash Torus Knots"

knot = torus_knot(2, 3)
graph = dbc_spinner(dcc_graph(id = "graph", figure = plot(knot, layout)))

slider_m = dcc_slider(
    id = "slider_m",
    min = 1,
    max = 7,
    step = 1,
    value = 3,
    marks = Dict([i => ("m=$(i)") for i = 1:10]),
)
slider_n = dcc_slider(
    id = "slider_n",
    min = 1,
    max = 7,
    step = 1,
    value = 2,
    marks = Dict([i => ("n=$(i)") for i = 1:10]),
)

input = dbc_row([dbc_col(slider_m, width = Dict("size" => 3, "order" => "first", "offset" => 3)),
 dbc_col(slider_n, width = Dict("size" => 3, "order" => "last", "offset" => 0))])
app.layout = dbc_container(
    [
        html_h1(html_center("Torus Knots"))
        html_br()
        description
        html_center(graph)
        html_center(input)
        html_br()
        html_br()
    ],
)

callback!(
    app,
    Output("graph", "figure"),
    Input("slider_m", "value"),
    Input("slider_n", "value"),
) do m, n
    knot = torus_knot(m, n)
    return plot(knot, layout)
end
run_server(app, "0.0.0.0", debug = true)


function gcd(x,y)
    if x == 0
        return y
    else
        return gcd(y%x,x)
    end
end
function find(arr,n)
    r = arr[1]
    for i=2:n
        r = gcd(arr[i],r)
    end
    return r
end
v = [8 16 8 4]
len = length(v)
println("The GCD of the array is = ",find(v,len))


@test norm(integrate(x -> 1 / x, 1.0, 2.0) - log(2)) < 10e-8
@test norm(integrate(x -> -sin(x), 0.0, pi) - (cos(pi) - cos(0.0))) < 10e-8

r = integrate(x -> 1, 0, 1)
@test norm(r - 1) < 10e-8

r = integrate(x -> x, 0, 1)
@test norm(r - 0.5) < 10e-8

r = integrate(x -> x * x, 0, 1)
@test norm(r - 1 / 3) < 10e-8

r = integrate(sin, 0, pi)
@test norm(r - 2) < 10e-8

r = integrate(cos, 0, pi)
@test norm(r - 0) < 10e-8

r = integrate(x -> sin(x)^2 + sin(x)^2, 0, pi)
@test norm(r - pi) < 10e-8

# Nice example, but requires Distributions
# require("Distributions")
# using Distributions
# r = integrate(x -> pdf(Normal(0.0, 1.0), x), -10, 10)
# @test norm(1 - r) < 10e-8

r = integrate(x -> 1 / x, 0.01, 1)
@test norm(r - 4.60517) < 10e-7

r = integrate(x -> cos(x)^8, 0, 2 * pi)
@test norm(r - 35 * pi / 64) < 10e-7

r = integrate(x -> sin(x) - sin(x^2) + sin(x^3), pi, 2 * pi)
@test norm(r - (-1.830467)) < 10e-7

# Monte Carlo integration tests
r = integrate(x -> sin(x), 0, pi, :monte_carlo)
@test norm(r - 2) < 10e-1


"""
    Fibonacci{T} <: UnivariateAlgorithm

The Fibonacci search algorithm optimize a univariate function by iteratively shrinking the
initial bracket around the minimum. The ratio is computed from the Fibonacci sequence.
"""
struct Fibonacci{T} <: UnivariateAlgorithm
    ϵ::T

    function Fibonacci{T}(ϵ) where {T}
        0 < ϵ < 1 ? new{T}(ϵ) : error("`ϵ` must be in the interval (0, 1)")
    end
end

"""
    Fibonacci([ϵ=0.01])

Initialize Fibonacci search algorithm with optional argument `ϵ` in the interval (0, 1)
to control how close the last iterate is to the previous one. Refer to Kochenderfer and
Wheeler's "Algorithms for Optimization" for more information.
"""
Fibonacci(ϵ=0.01) = Fibonacci{typeof(ϵ)}(ϵ)

function _optimize(f, lower::T, upper::T, alg::Fibonacci;
        reltol, abstol, maxiter) where {T}
    ϵ = T(alg.ϵ)
    s = (1 - √5) / (1 + √5)
    p = 1 / (φ * (1 - s^(maxiter + 1)) / (1 - s^maxiter))
    x = T(p * lower + (1 - p) * upper)
    yx = f(x)
    converged = false
    iter = 1
    while true
        x_tol = reltol * abs(x) + abstol
        if abs(upper - lower) < 2x_tol
            converged = true
            break
        end
        iter == maxiter && break
        iter += 1
        if iter == maxiter
            new_x = T(ϵ * upper + (1 - ϵ) * x)
        else
            new_x = T(p * upper + (1 - p) * lower)
        end
        new_yx = f(new_x)
        if yx > new_yx
            lower, x, yx = x, new_x, new_yx
        else
            lower, upper = new_x, lower
        end
        p = 1 / (φ * (1 - s^(maxiter - iter + 2)) / (1 - s^(maxiter - iter + 1)))
    end
    return Solution(converged, iter, x, yx)
end


export l1_distance, l2_distance, squared_l2_distance, linf_distance, lp_distance

""" l1_distance computes the Manhattan's distance """
function l1_distance(a::AbstractVector{T}, b::AbstractVector{T})::Float64 where {T <: Real}
    d::T = zero(T)

    @fastmath @inbounds @simd for i = 1:length(a)
	    m = a[i] - b[i]
        d += ifelse(m > 0, m, -m)
    end

    return d
end

""" l2_distance computes the Euclidean's distance """
function l2_distance(a::AbstractVector{T}, b::AbstractVector{T})::Float64 where {T <: Real}
    d::T = zero(T)

    @fastmath @inbounds @simd for i = 1:length(a)
        m = a[i] - b[i]
        d += m * m
    end

    return sqrt(d)
end

""" squared_l2_distance computes the Euclidean's distance but squared """
function squared_l2_distance(a::AbstractVector{T}, b::AbstractVector{T})::Float64 where {T <: Real}
    d::T = zero(T)

    @fastmath @inbounds @simd for i = 1:length(a)
        m = a[i] - b[i]
        d += m * m
    end

    return d
end

""" linf_distance computes the max distance """
function linf_distance(a::AbstractVector{T}, b::AbstractVector{T})::Float64 where {T <: Real}
   d::T = zero(T)

    @fastmath @inbounds @simd for i = 1:length(a)
        m = abs(a[i] - b[i])
        d = max(d, m)
    end

    return d
end

"""
lp_distance creates a function that computes computes generic Minkowski's distance
"""
function lp_distance(p_::Real)
    p::Float64 = convert(Float64, p_)
    invp = 1.0 / p

    function _lp(a::AbstractVector{T}, b::AbstractVector{T})::Float64 where T
        d::T = zero(T)

        @fastmath @inbounds @simd for i = 1:length(a)
            m = abs(a[i] - b[i])
            d += m ^ p
        end

        d ^ invp
    end
end


# # Chebyshev center
# Boyd & Vandenberghe, "Convex Optimization"
# Joëlle Skaf - 08/16/05
#
# Adapted for Convex.jl by Karanveer Mohan and David Zeng - 26/05/14
#
# The goal is to find the largest Euclidean ball (i.e. its center and
# radius) that lies in a polyhedron described by affine inequalites in this
# fashion: $P = \{x : a_i'*x \leq b_i, i=1,\ldots,m \}$ where $x \in \mathbb{R}^2$.

using Convex, LinearAlgebra, SCS

# Generate the input data
a1 = [ 2;  1];
a2 = [ 2; -1];
a3 = [-1;  2];
a4 = [-1; -2];
b = ones(4, 1);

# Create and solve the model
r = Variable(1)
x_c = Variable(2)
p = maximize(r)
p.constraints += a1' * x_c + r * norm(a1, 2) <= b[1];
p.constraints += a2' * x_c + r * norm(a2, 2) <= b[2];
p.constraints += a3' * x_c + r * norm(a3, 2) <= b[3];
p.constraints += a4' * x_c + r * norm(a4, 2) <= b[4];
solve!(p, () -> SCS.Optimizer(verbose=0))
p.optval

# Generate the figure
x = range(-1.5, stop=1.5, length=100);
theta = 0:pi/100:2*pi;
using Plots
plot(x, x -> -x * a1[1] / a1[2] + b[1] / a1[2])
plot!(x, x -> -x * a2[1]/ a2[2] + b[2] / a2[2])
plot!(x, x -> -x * a3[1]/ a3[2] + b[3] / a3[2])
plot!(x, x -> -x * a4[1]/ a4[2] + b[4] / a4[2])
plot!(evaluate(x_c)[1] .+ evaluate(r) * cos.(theta), evaluate(x_c)[2] .+ evaluate(r) * sin.(theta), linewidth = 2)
plot!(title ="Largest Euclidean ball lying in a 2D polyhedron", legend = nothing)


## This example comes from https://diffeqflux.sciml.ai/dev/Flux/

using ComponentArrays
using OrdinaryDiffEq
using Plots
using UnPack

using DiffEqFlux: sciml_train
using Flux: glorot_uniform, ADAM, σ, relu
using Optim: LBFGS

struct MeasurementNoise{T}
    sigma::T
end
Base.:(+)(array, mn::MeasurementNoise) = array .+ randn(size(array)...).*mn.sigma

# Problem setup
u0 = Float32[2.; 0.]
datasize = 50
tspan = (0.0f0, 7f0)
tspan2 = (0.0f0, 25f0)

# Make truth data
function trueODEfunc(du, u, p, t)
    true_A = [-0.1 2.0; -2.0 -0.1]
    du .= ((u.^3)'true_A)'
end

t = range(tspan[1], tspan[2], length = datasize)
# t = Float32.(vcat(range(0.0, 0.9, length=10), 10 .^ range(log10(tspan[1]+1), log10(tspan[2]), length=datasize-10)))
t = Float32.([0; 10 .^ range(log10(tspan[1] + 0.01), log10(tspan[2]), length=datasize-1)])
prob = ODEProblem(trueODEfunc, u0, tspan2)
ode_sol = solve(prob, Tsit5())
ode_data = Array(ode_sol(t)) + MeasurementNoise(0.1)

# Function for creating neural layer components
neural_layer(in, out) = ComponentArray{Float32}(W=glorot_uniform(out, in), b=zeros(out))

# Dense neural layer function
dense(layer, activation=identity) = u -> activation.(layer.W * u + layer.b)

# Neural ODE function
dudt(u, p, t) = u.^3 |> dense(p.L1, σ) |> dense(p.L2)

prob = ODEProblem(dudt, u0, tspan2)

# Create optimization parameter vector
n_hidden = 50
layers = (L1=neural_layer(2, n_hidden), L2=neural_layer(n_hidden, 2))
θ = ComponentArray(u=u0, p=layers)

# Prediction and loss functions
predict_n_ode(θ) = Array(solve(prob, Tsit5(), u0=θ.u, p=θ.p, saveat=t))
full_sol(θ) = solve(prob, Tsit5(), u0=θ.u, p=θ.p)

function loss_n_ode(θ)
    pred = predict_n_ode(θ)

    loss = sum(abs2, ode_data .- pred)/datasize + 0.1*(sum(abs, θ.p)/length(θ.p))
    return loss, pred
end
loss_n_ode(θ)

# anim = Animation()

 # Callback function to observe training
cb = function (θ, loss, pred; doplot=false)
    display(loss)
    # plot current prediction against data
    train_sol = full_sol(θ)

    pl_1 = plot(train_sol, vars=1)
    plot!(pl_1, ode_sol, vars=1)
    scatter!(pl_1, t, pred[1,:])
    scatter!(pl_1, t, ode_data[1,:], legend=false)

    pl_2 = plot(train_sol, vars=2)
    plot!(pl_2, ode_sol, vars=2)
    scatter!(pl_2, t, pred[2,:])
    scatter!(pl_2, t, ode_data[2,:], legend=false)

    pl_3 = plot(train_sol, vars=(1,2), label = "prediction")
    plot!(pl_3, ode_sol, vars=(1,2), label = "truth")
    scatter!(pl_3, pred[1,:], pred[2,:], label = "predicted data")
    scatter!(pl_3, ode_data[1,:], ode_data[2,:], label = "measured data")
    plot!(pl_3, hcat(pred[1,:], ode_data[1,:])', hcat(pred[2,:], ode_data[2,:])', label = false, color=:lightgray, legend=:bottomright)

    display(plot(plot(pl_1, pl_2, layout=(2,1), size=(400,500)), pl_3, layout=(1,2), size=(950,500)))
    # frame(anim)
    return false
end

# Display the ODE with the initial parameter values.
cb(θ, loss_n_ode(θ)...)

data = Iterators.repeated((), 1000)

res1 = sciml_train(loss_n_ode, θ, ADAM(0.05); maxiters=100, save_best=true, cb=cb)
cb(res1.minimizer, loss_n_ode(res1.minimizer)...; doplot=true)

res2 = sciml_train(loss_n_ode, res1.minimizer, LBFGS(), cb=cb)
cb(res2.minimizer, loss_n_ode(res2.minimizer)...; doplot=true)

# gif(anim, "DiffEqFlux.gif", fps=15)


# # Maximum a Posteriori

using Distributions
using HMMBase
using PyPlot
using Seaborn

rc("axes", xmargin = 0) # hide
set_style("whitegrid")  # hide

# Let's consider a simple time series with one outlier:
y = rand(1000)
y[100] = 10000
figure(figsize = (9,2)) # hide
plot(y)
gcf() # hide
#-

# An MLE approach for the observations distributions parameters
# may fail with a singularity (variance = 0) if an outlier
# becomes the only observation associated to some state:
hmm = HMM([0.9 0.1; 0.1 0.9], [Normal(0,1), Normal(5,1)])
try
    fit_mle(hmm, y, display=:iter);
catch e
    println(e)
end

# We can avoid this by putting a prior on the variance:
import ConjugatePriors: InverseGamma, NormalKnownMu, posterior_canon
import StatsBase: Weights

function fit_map(::Type{<:Normal}, observations, responsibilities)
    μ = mean(observations, Weights(responsibilities))

    ss = suffstats(NormalKnownMu(μ), observations, responsibilities)
    prior = InverseGamma(2,1)
    posterior = posterior_canon(prior, ss)
    σ2 = mode(posterior)

    Normal(μ, sqrt(σ2))
end

hmm, hist = fit_mle(hmm, y, estimator=fit_map, display=:iter)
figure(figsize = (4,3)) # hide
plot(hist.logtots)
xlabel("Iteration") # hide
ylabel("Log-likelihood") # hide
gcf() # hide
#-

hmm.B