function [x,err,it] = perform_conjugate_gradient(A,y)

    """
     perform_conjugate_gradient - perform (bi)-conjugate gradient

         [x,err,k] = perform_conjugate_gradient(A,y,options);

       Solves for A*x=y.
       Works both for vector x,y and matrix x,y (parralel soving of many
       linear equations).

       Important: the algorithm assumes that the matrix is symmetric definite
       positive. If it is not the case, then you must set options.is_sdp=0,
       and the algorithm will use a bi-conjugate gradient descent (x2 slower).

       A can be a matrix or a callback function y=A(x,options).
       In this case, you have to set options.ncols as the number of columns of
       A (if it is a callback).
       In the non-symmetric case, then options.transpose=1 for the computation
       of A*x and options.transpose=-1 for the computation of A'*x.

       err monitors the decreasing of |A*x-y|.
       k is the total number of iterations.

       You can set:
           options.x is an initial guess
           options.epsilon is maximum error
           options.niter_max is maximum number of iterations

       Copyright (c) 2007 Gabriel Peyre
    """

options.null = 0;
niter = 100
epsilon = getoptions(options, 'epsilon', 1e-5);
is_sdp = getoptions(options, 'is_sdp', 1);
x = getoptions(options, 'x', []);

use_callback = 0;
if not(isnumeric(A))
    use_callback = 1;
end
if not(isempty(x))
    x = options.x;
else
    if use_callback==0
        x = zeros(size(A,2),1);
    else
        if isfield(options, 'ncols')
            x = zeros(options.ncols, 1);
        else
            error('You have to specify options.ncols');
        end
    end
end

if norm(y(:), 'fro') == 0
    normb = epsilon;
else
    normb = epsilon; % * sum(y(:).^2);
end

if use_callback==0
    r  = y - A*x;
else
    options.transpose = 1;
    r  = y - feval(A,x,options);
end
p = r;
if is_sdp==0
    rr = r;
    pp = p;
end
r0 = sum(r(:).^2);

err = [sum(r0)];
for it=1:niter

    % auxiliary vector
    if use_callback==0
        w  = A*p;
    else
        options.transpose = 1;
        % options.transpose = -1;
        w  = feval(A,p,options);
    end

    if is_sdp==1
        d = sum(p(:) .* w(:));
    else
        d = sum(pp(:) .* w(:));
    end
    I = find(abs(d)<eps); d(I) = 1;
    alpha = repmat( r0 ./ d, [size(x,1) 1] );              % found optimal alpha in line search
    x = x + alpha.*p;                       % new guess
    r = r - alpha.*w;                       % the residual is in fact r=b-A*x
    if is_sdp==0
        if use_callback==0
            w  = A'*pp;
        else
            options.transpose = -1;
            w  = feval(A,pp,options);
        end
        rr = rr-alpha.*w;
    end

    rprev = r0;                             % save norm of the old residual
    if is_sdp==0
        r0 = sum(r.*rr);
    else
        r0 = sum(r.^2);                         % compute norm of new residual
    end

    err(end+1) = sqrt( sum(r.^2) );
    if err(end)<normb
        return;
    end

    % search direction
    beta = r0./rprev;
    p = r + repmat(beta, [size(x,1) 1]).*p;
    if is_sdp==0
        pp = rr + repmat(beta, [size(x,1) 1]).*pp;
    end

end


using LinearAlgebra

# k_local(Elastic Modulus,thickness,height,length,number of nodes,row one, row two, column one, column two)

#creates local stiffness matrix and maps to global
function k_local(E,t,h,l,n,r_1,r_2,c_1,c_2)
    k_global = zeros(n,n)
    A = t*h
    k = ((E*A)/l)
    k_global[r_1,c_1] = k
    k_global[r_1,c_2] = -k
    k_global[r_2,c_1] = -k
    k_global[r_2,c_2] = k
    return k_global
end

# constants from problem statement
E = 29*10^6
t = 1/16
h14 = 5
h23 = 2
l1 = 1
l23 = 4
l4 = 2

# combining all globally mapped matrices
kG = k_local(E,t,h14,l1,4,1,2,1,2)+
     k_local(E,t,h23,l23,4,2,3,2,3)+
     k_local(E,t,h23,l23,4,2,3,2,3)+
     k_local(E,t,h14,l4,4,3,4,3,4)
kG[1,:] = [1 0 0 0] #adjustment for Reaction from fixed end

# Force Vector
f = Float64[0 0 0 800]
F = transpose(f)

# Displacement vector
u = kG\F
println("Nodal displacement vector:")
display(u)

# Strain Function
ϵ(x2,x1,l) = (x2 - x1)/l

#Strain Vector
ϵM = [ϵ(u[2],u[1],l1)
      ϵ(u[3],u[2],l23)
      ϵ(u[3],u[2],l23)
      ϵ(u[4],u[3],l4) ]
println("Element strain vector")
display(ϵM)

# Stress Function
σ(E,ϵ) = E*ϵ

# Stress Vector
σM = [σ(E,ϵM[1])
      σ(E,ϵM[2])
      σ(E,ϵM[3])
      σ(E,ϵM[4])]
println("Element Stress Vector")
display(σM)


########################################################################
function ppca(X; Q=1)
########################################################################

    D, N = size(X)

    μ = vec(mean(X, dims=2))

    JITTER = 1e-8

    @printf("Running PPCA for %d number of data items of dimension %d, projected to %d components\n", N, D, Q)

    #---------------------------------------------
    function marginalLogLikelihood(W, σ)
    #---------------------------------------------

        sum(logpdf(MvNormal(μ, W*W' + σ*σ*I + JITTER*I), X))

    end

    #---------------------------------------------
    function unpack(param)
    #---------------------------------------------

        local W = reshape(param[1:end-1], D, Q)

        local σ = exp(param[end])

        return W, σ

    end

    #---------------------------------------------
    function objective(param)
    #---------------------------------------------

        @assert(length(param) == D*Q + 1)

        local W, σ = unpack(param)

        return -1.0 * marginalLogLikelihood(W, σ)

    end

    #---------------------------------------------
    # Run optimiser
    #---------------------------------------------

    opt    = Optim.Options(show_trace = true, iterations = 10_000)

    result = optimize(objective, [randn(D*Q); randn()*3], LBFGS(), opt, autodiff=:forward)

    W, σ   = unpack(result.minimizer)

    @show W, μ, σ

    #---------------------------------------------
    # Define projections
    #---------------------------------------------

    M = W'*W  + σ*σ*I # eq. (12.41) in Bishop

    proj(x::Array{T,1} where T<:Real) = vec(M \ (W'*(x - μ)))[1]

    fwd(z::Real) = vec(W*z + μ)

    return W, μ, σ, proj, fwd

end


"""
    CosineKernel()

Cosine kernel.

# Definition

For inputs ``x, x' \\in \\mathbb{R}^d``, the cosine kernel is defined as
```math
k(x, x') = \\cos(\\pi \\|x-x'\\|_2).
```
"""
struct CosineKernel <: SimpleKernel end

kappa(::CosineKernel, d::Real) = cospi(d)

metric(::CosineKernel) = Euclidean()

Base.show(io::IO, ::CosineKernel) = print(io, "Cosine Kernel")


# Moreau envelope of box indicator

f = IndBox(-1, 1)
g = MoreauEnvelope(f, 1e-2)

predicates_test(g)

@test ProximalOperators.is_smooth(g) == true
@test ProximalOperators.is_quadratic(g) == false
@test ProximalOperators.is_set(g) == false

x = [1.0, 2.0, 3.0, 4.0, 5.0]

grad_g_x, g_x = gradient(g, x)

# Moreau envelope of L2 norm = (circulant) Huber loss (for some parameters)

rho = 1e-1
mu = 1e0
f = NormL2(mu)
g = MoreauEnvelope(f, rho)
h = HuberLoss(rho, mu/rho)

predicates_test(g)

@test ProximalOperators.is_smooth(g) == true
@test ProximalOperators.is_quadratic(g) == false
@test ProximalOperators.is_set(g) == false

x = [1.0, 2.0, 3.0, 4.0, 5.0]

@test abs(g(x) - h(x)) <= 1e-12

grad_g_x, g_x = gradient(g, x)
grad_h_x, h_x = gradient(h, x)

@test abs(g_x - g(x)) <= 1e-12
@test abs(h_x - h(x)) <= 1e-12
@test norm(grad_g_x - grad_h_x, Inf) <= 1e-12


using NumericalMethodsforEngineers, IterativeSolvers

a = [16. 4. 8.; 4. 5. -4.; 8. -4. 22.]
b = [4., 2., 5.]

f = cholesky(a)
y = f.L \ b
c = f.U \ y

# p_2_11 version

x0 = [1., 1., 1.]
tol = 1.0e-5
limit = 1000

function steepest_descent(a::Matrix{Float64}, b::Vector{Float64}, x0::Vector{Float64};
  tol=1.0e-10, limit=1000, trace=false)

  x = copy(x0)
  trace && println("\nFirst few iterations:")
  r = b - a * x
  iters = 0
  while true
    iters += 1
    u = a * r
    α = r⋅r / dot(r, u)
    xnew = x + α * r
    r -= α * u
    if trace && iters <= 5
      println("After $iters iterations: x = $x")
    end
    (checkit(xnew, x, tol) || (iters == limit)) && break
    x = xnew
  end
  converged = iters <= limit
  iters, converged, x
end

iters, converged, x = steepest_descent(a, b, x0; tol=tol, limit=limit, trace=true)
println("\nSolution Vector: $x")
println("Converged: $(iters < limit)")
println("Number of iters: $(iters)")

@test round.(x; digits=3) == round.(c; digits=3)

a = [16. 4. 8.; 4. 5. -4.; 8. -4. 22.]
b = [4., 2., 5.]

tol = 1.0e-10
iters = 20
x = cg(a, b; maxiter=iters);

println("\nSolution Vector: $x")

@test round.(x; digits=6) == round.(c; digits=6)


### A Pluto.jl notebook ###
# v0.12.6

using Markdown
using InteractiveUtils

# ╔═╡ de7a5e8a-1ee4-11eb-2bbf-0f0eb3e489ec
using Pkg, DrWatson

# ╔═╡ e4eeb13a-1ee4-11eb-0113-91f9a9c3b659
begin
	@quickactivate "StatisticsWithJuliaPlutoNotebooks"
	using Plots
end

# ╔═╡ ac3ceabe-1ee4-11eb-3935-313e96aafd62
md"## Listing 2.9"

# ╔═╡ c4bef1a4-1ee9-11eb-0b58-8be033f6fbad
occupancyAnalytic(n,r) =  sum([(-1)^k*binomial(n,k)*(1 - k/n)^r for k in 0:n])

# ╔═╡ 8ce90c92-1eee-11eb-21f7-f78acba42f4a
function occupancyMC(n,r,N)
    fullCount = 0
    for _ in 1:N
        envelopes = zeros(Int,n)
        for k in 1:r
            target = rand(1:n)
            envelopes[target] += 1
        end
        numFilled = sum(envelopes .> 0)
        if numFilled == n
            fullCount += 1
        end
    end
    return fullCount/N
end

# ╔═╡ 8ce947c0-1eee-11eb-0f4e-49531a7b01c6
begin
	max_n, N, Kvals = 100, 10^3, [2,3,4]

	analytic = [[occupancyAnalytic(big(n),big(k*n)) for n in 1:max_n] for k in Kvals]
	monteCarlo = [[occupancyMC(n,k*n,N) for n in 1:max_n] for k in Kvals]

	plot(1:max_n, analytic, c=[:blue :red :green],
		label=["K=2" "K=3" "K=4"])
	scatter!(1:max_n, monteCarlo, mc=:black, shape=:+,
		label="", xlims=(0,max_n),ylims=(0,1),
		xlabel="n Envelopes", ylabel="Probability", legend=:topright)
end

# ╔═╡ ea086874-1eeb-11eb-2cb0-4186697d4294
md"## End of listing 2.9"

# ╔═╡ Cell order:
# ╟─ac3ceabe-1ee4-11eb-3935-313e96aafd62
# ╠═de7a5e8a-1ee4-11eb-2bbf-0f0eb3e489ec
# ╠═e4eeb13a-1ee4-11eb-0113-91f9a9c3b659
# ╠═c4bef1a4-1ee9-11eb-0b58-8be033f6fbad
# ╠═8ce90c92-1eee-11eb-21f7-f78acba42f4a
# ╠═8ce947c0-1eee-11eb-0f4e-49531a7b01c6
# ╟─ea086874-1eeb-11eb-2cb0-4186697d4294


module FinSight
using Roots: find_zero
using Dates: DateTime, Day

"""
Calculates the net present value of an investment by using a discount rate and a
series of future payments (negative values) and income (positive values).

rate - the rate of discount over the length of one period.
values - arguments representing the payments and income.

From https://support.microsoft.com/en-us/office/npv-function-8672cb67-2576-4d07-b67b-ac28acf2a568
"""
function npv(rate::Real, values::AbstractVector{T}) where T <:Real
    @assert length(values) > 0
    out = 0
    for (i, val) in enumerate(values)
        out += val / (1 + rate)^i
    end
    return out
end
export npv

"""
Returns the internal rate of return for a series of cash flows represented by
the numbers in values. These cash flows do not have to be even, as they would be
for an annuity.

values - An array or a reference to cells that contain numbers for
which you want to calculate the internal rate of return.
rate_guess -  A number that you guess is close to the result of IRR

From https://support.microsoft.com/en-us/office/irr-function-64925eaa-9988-495b-b290-3ad0c163c1bc
"""
function irr(values::AbstractVector{T}, rate_guess::Real = 0.1) where T <:Real
    @assert length(values) > 0
    rate = find_zero(x -> npv(x, values),  rate_guess, )
    return rate
end
export irr

"""
Returns the net present value for a schedule of cash flows that is not
necessarily periodic. To calculate the net present value for a series of cash
flows that is periodic, use the NPV function.
"""
function xnpv(rate::Real, values::AbstractVector{T}, dates::Vector{DateTime}) where T <:Real
    @assert length(values) > 0
    @assert length(values) == length(dates)
    out = 0
    for (i, val) in enumerate(values)
        exponent = Day(dates[i] - first(dates)) / Day(365)
        out += val / (1 + rate)^exponent
    end
    return out
end
export xnpv

"""
Returns the internal rate of return for a schedule of cash flows that is not
necessarily periodic. To calculate the internal rate of return for a series of
periodic cash flows, use the IRR function.
"""
function xirr(values::AbstractVector{T}, dates::Vector{DateTime}, rate_guess::Real = 0.1) where T <:Real
    @assert length(values) > 0
    @assert length(values) == length(dates)
    rate = find_zero(x -> xnpv(x, values, dates),  rate_guess)
    return rate
end
export xirr

end


# ------------------------------------------------------------------
# Licensed under the MIT License. See LICENSE in the project root.
# ------------------------------------------------------------------

"""
    ∠(A, B, C)

Angle ∠ABC between rays BA and BC.
See https://en.wikipedia.org/wiki/Angle.

Uses the two-argument form of `atan`.
See https://en.wikipedia.org/wiki/Atan2.

## Example

```julia
∠(Point(1,0), Point(0,0), Point(0,1)) == π/2
```
"""
function ∠(A::P, B::P, C::P) where {P<:Point{2}}
  BA = A - B
  BC = C - B
  atan(BA × BC, BA ⋅ BC)
end


#############################################################
# Fornberg algorithm

# This implements the Fornberg (1988) algorithm (https://doi.org/10.1090/S0025-5718-1988-0935077-0)
# to obtain Finite Difference weights over arbitrary points to arbitrary order

function calculate_weights(order::Int, x0::T, x::AbstractVector) where T<:Real
    #=
        order: The derivative order for which we need the coefficients
        x0   : The point in the array 'x' for which we need the coefficients
        x    : A dummy array with relative coordinates, eg. central differences
               need coordinates centred at 0 while those at boundaries need
               coordinates starting from 0 to the end point

        The approximation order of the stencil is automatically determined from
        the number of requested stencil points.
    =#
    N = length(x)
    @assert order < N "Not enough points for the requested order."
    M = order
    c1 = one(T)
    c4 = x[1] - x0
    C = zeros(T, N, M+1)
    C[1,1] = 1
    @inbounds for i in 1 : N-1
        i1 = i + 1
        mn = min(i, M)
        c2 = one(T)
        c5 = c4
        c4 = x[i1] - x0
        for j in 0 : i-1
            j1 = j + 1
            c3 = x[i1] - x[j1]
            c2 *= c3
            if j == i-1
                for s in mn : -1 : 1
                    s1 = s + 1
                    C[i1,s1] = c1*(s*C[i,s] - c5*C[i,s1]) / c2
                end
                C[i1,1] = -c1*c5*C[i,1] / c2
           end
            for s in mn : -1 : 1
                s1 = s + 1
                C[j1,s1] = (c4*C[j1,s1] - s*C[j1,s]) / c3
            end
            C[j1,1] = c4 * C[j1,1] / c3
        end
        c1 = c2
    end
    #=
        This is to fix the problem of numerical instability which occurs when the sum of the stencil_coefficients is not
        exactly 0.
        https://scicomp.stackexchange.com/questions/11249/numerical-derivative-and-finite-difference-coefficients-any-update-of-the-fornb
        Stack Overflow answer on this issue.
        http://epubs.siam.org/doi/pdf/10.1137/S0036144596322507 - Modified Fornberg Algorithm
    =#
    _C = C[:,end]
    _C[div(N,2)+1] -= sum(_C)
    return _C
end


using DataFrames, Query, Turing, StatsFuns

using RDatasets

cars = dataset("datasets", "cars")

@model model7_20(dist, speed) = begin
    σ ~ Exponential(1)
    α ~ Normal(0, 100)
    β ~ Normal(0, 10)
    for i ∈ eachindex(dist)
        dist[i] ~ Normal(α + β * speed[i], σ)
    end
end

dist = cars.Dist
speed = cars.Speed

model = model7_20(dist, speed)

m7_20 = sample(model, NUTS(), 2000, save_state = true)

describe(m7_20)

loglik =
[logprob"dist = dist[i] | speed = speed[i], chain = m7_20" for i ∈ eachindex(dist)]

ll = DataFrame(loglik)

function WAIC(ll::DataFrame)
    n_samples, n_obs = size(ll)
    logsums = [logsumexp(col) for col ∈ eachcol(ll)]
    lppd = logsums .- log(n_samples)
    pWAIC = [var(col) for col ∈ eachcol(ll)]
    sum_lppd = sum(lppd)
    sum_pWAIC =sum(pWAIC)
    waic = -2 * (sum_lppd - sum_pWAIC)
    waic_vec = -2 .* (lppd .- pWAIC)
    se = sqrt(n_obs * var(waic_vec))

    return (waic = waic, se = se, lppd = sum_lppd, pwaic = sum_pWAIC)
end

waic, se, lppd, pWAIC = WAIC(ll)


#!/usr/bin/env julia
#
# Jerzy Wroczyński (nr. indeksu 250075)
# Obliczenia Naukowe, Laboratorium
# Lista 1, Zadanie 7.
#

fl = Float64

function f(x::fl)::fl
    return fl(sin(x) + cos(3*x))
end

function f_derivative(x::fl)::fl
    return fl(cos(x) - 3 * sin(3*x))
end

# break out this part of the approximate function for later use
function _approximate_derivative_internal(func, x::fl, h::fl)::fl
    return fl(func(x + h) - func(x))
end
function approximate_derivative_at_given_point(func, x::fl, h::fl)::fl
    return fl( _approximate_derivative_internal(func, x, h) / h )
end

# calculate the results and print them out
for n in -25:-1:-32
    h = fl(2.0^n)
    x = fl(1.0)
    appr = approximate_derivative_at_given_point(f, x, h)
    exact = f_derivative(x)

    println(
        "n = ", n, "\n",
        "h                         = ", h, "\n",
        "f(x + h) - f(x)           = ", _approximate_derivative_internal(f, x, h), "\n",
        "\\tilde{f’}(x)             = ", appr, "\n",
        "f’(x)                     = ", exact, "\n",
        "|f’(x) - \\tilde{f’}(x)|   = ", abs( fl(exact - appr) ),
        "\n"
    )
end


using QuadGK

"""
    AMFMcomp(in::Vector{Float64}, out::Vector{Complex{Float64}})
Creates an AMFM component with parameters `a`, `ω`, and `φ`.
    ψ =  a(t) exp[j(∫ω(τ)dt + φ)]

The input `t` must be a vector of Float64 and the out `ψ` will be a vector of
Complex{Float64} the same length as `t`.

```julia
julia> a₀(t)=1; ω₀(t)=1; φ₀ = 0; ψ₀ = AMFMcomp(a₀,ω₀,φ₀)
AMFMcomp(a₀, ω₀, 0)

julia> t = Array(0.0:0.5:1.0); ψ₀(t)
3-element Array{Complex{Float64},1}:
  1.0 + 0.0im
 -1.0 + 1.2246467991473532e-16im
  1.0 - 2.4492935982947064e-16im
```
"""

#Define type AMFMcomp as a struct of IA, IF, and phase reference
struct AMFMcomp
#References: Sandoval, Steven, and Phillip L. De Leon. "The Instantaneous Spectrum: A General Framework for Time-Frequency Analysis." IEEE Transactions on Signal Processing 66.21 (2018): 5679-5693.
  a::Function
  ω::Function
  φ::Real
end

#Unspecifed phase references are assumed to be zero
AMFMcomp(a, ω) = AMFMcomp(a, ω, 0.0)

#Method for type AMFMcomp when called with a time index
function (ψ::AMFMcomp)(t::Vector{Float64})
#References: Sandoval, Steven, and Phillip L. De Leon. "The Instantaneous Spectrum: A General Framework for Time-Frequency Analysis." IEEE Transactions on Signal Processing 66.21 (2018): 5679-5693.
  a, ω, φ = ψ.a, ψ.ω, ψ.φ
  A = quadgk.(ω, 0.0, t)
  θ = [ A[i][1] for i in 1:length(t) ]
  a.(t) .* exp.( 1im .* ( θ .+ φ ) )
end

function (ψ::AMFMcomp)(t::Float64)
  return (ψ::AMFMcomp)([t::Float64])[1]
end

function (ψ::AMFMcomp)(t::StepRangeLen)
  return (ψ::AMFMcomp)(collect(t))
end

#Construction of type AMFMcomp from a cannonical triplet
function AMFMcomp(triplet::Tuple{Function, Function, Real})
#References: Sandoval, Steven, and Phillip L. De Leon. "The Instantaneous Spectrum: A General Framework for Time-Frequency Analysis." IEEE Transactions on Signal Processing 66.21 (2018): 5679-5693.
      return AMFMcomp(triplet[1],triplet[2],triplet[3])
end


# indicator of nonnegative orthant

export IndNonnegative

"""
**Indicator of the nonnegative orthant**

    IndNonnegative()

Returns the indicator of the set
```math
C = \\{ x : x \\geq 0 \\}.
```
"""
struct IndNonnegative <: ProximableFunction end

is_separable(f::IndNonnegative) = true
is_convex(f::IndNonnegative) = true
is_cone(f::IndNonnegative) = true

function (f::IndNonnegative)(x::AbstractArray{R}) where R <: Real
    for k in eachindex(x)
        if x[k] < 0
            return R(Inf)
        end
    end
    return R(0)
end

function prox!(y::AbstractArray{R}, f::IndNonnegative, x::AbstractArray{R}, gamma=R(1)) where R <: Real
    for k in eachindex(x)
        if x[k] < 0
            y[k] = R(0)
        else
            y[k] = x[k]
        end
    end
    return R(0)
end

fun_name(f::IndNonnegative) = "indicator of the Nonnegative cone"
fun_dom(f::IndNonnegative) = "AbstractArray{Real}"
fun_expr(f::IndNonnegative) = "x ↦ 0 if all(0 ⩽ x), +∞ otherwise"
fun_params(f::IndNonnegative) = "none"

function prox_naive(f::IndNonnegative, x::AbstractArray{R}, gamma=R(1)) where R <: Real
    y = max.(R(0), x)
    return y, R(0)
end


PS-2

using Optim,DataFrames, CSV, HTTP,GLM,FreqTables

#Question 1

f(x) = -x[1]^4-10x[1]^3-2x[1]^2-3x[1]-2
	minusf(x) = x[1]^4+10x[1]^3+2x[1]^2+3x[1]+2
	startval = rand(1)   # random starting value
	result = optimize(minusf, startval, BFGS())
	println(result)

    #Question 2

    url = "https://raw.githubusercontent.com/OU-PhD-Econometrics/fall-2021/master/ProblemSets/PS1-julia-intro/nlsw88.csv"
	df = CSV.read(HTTP.get(url).body, DataFrame)
	X = [ones(size(df,1),1) df.age df.race.==1 df.collgrad.==1]
	y = df.married.==1

	function ols(beta, X, y)
		ssr = (y.-X*beta)'*(y.-X*beta)
		return ssr
	end

	beta_hat_ols = optimize(b -> ols(b, X, y), rand(size(X,2)), LBFGS(), Optim.Options(g_tol=1e-6, iterations=100_000, show_trace=true))
	println(beta_hat_ols.minimizer)

	bols = inv(X'*X)*X'*y
	df.white = df.race.==1
	bols_lm = lm(@formula(married ~ age + white + collgrad), df)

    #Question 3

    function logit(alpha, X, d)
		log_mat=[d[i]*log((1+exp(-(X*alpha)[i]))^-1)+(1-d[i])*log(1-(1+exp(-(X*alpha)[i]))^-1) for i in 1:size(d,1)]
	#     p=(1+exp(-X*alpha))^-1
		loglike=-sum(log_mat)

		return loglike
	end
	beta_loglike = optimize(b -> logit(b, X, y), rand(size(X,2)), LBFGS(), Optim.Options(g_tol=1e-6, iterations=100_000, show_trace=true))
	println(beta_loglike.minimizer)

    #Question 4

    df.white = df.race.==1
	α̂ = glm(@formula(married ~ age + white + collgrad), df, Binomial(), LogitLink())
	println(α̂)

    #Question 5

    	freqtable(df, :occupation) # note small number of obs in some occupations
	df = dropmissing(df, :occupation)
	df[df.occupation.>7,:occupation] .= 7
	freqtable(df, :occupation) # problem solved

	X = [ones(size(df,1),1) df.age df.race.==1 df.collgrad.==1]
	y = df.occupation

	function mnlogit(alpha, x, d)
		c=[i==j ? 1 : 0 for i in d , j in 1:6]
		log_mat=[log(exp.(x[i,:]'*alpha[:,j])/(1+sum(exp.(x[i,:]'*alpha)))) for i in 1:size(x,1) ,j in 1:6]
		loglike=-sum(c.*log_mat)

		return loglike
	end

	beta_mnlog = optimize(b -> mnlogit(b, X, y), rand(size(X,2),6), LBFGS(), Optim.Options(g_tol=1e-7, iterations=100_000, show_trace=true))
	println(beta_mnlog.minimizer)


@test inf(1.) == inf(Float64) == Inf
@test inf(Float16(1.)) == inf(Float16) == Inf16
@test_throws InexactError inf(Int)

for x = 0.:0.1:10.
    @test log1mexp(x) ≈ log(1 - exp(-x))
end

for x = -100.:0.
    @test log1pexp(x) ≈ log(1 + exp(x)) atol=1e-14
end

for x = 0.:100.
    @test log1pexp(x) ≈ x + log(exp(-x) + 1) atol=1e-14
end

for x = 0.:10., y = 0.:10.
    @test logadd(log(x), log(y)) ≈ log(x + y)
    @test logsub(log(x), log(y)) ≈ log(abs(x - y)) atol=1e-14
end

for x = 0.:10., y = 0.:10., z = 0.:10.
    @test logadd(log(x), log(y), log(z)) ≈ log(x + y + z)
end

for rep = 1 : 100
    x = 10rand(10)
    lx = log.(x)
    if logadd(lx...) ≈ log(sum(x))
        @test true
        @test logadd(lx[1],lx[2],lx[3],lx[4],lx[5],lx[6],lx[7],lx[8],lx[9],lx[10]) ≈ log(sum(x))
    else
        println("logadd reduce test failed, with x=$x")
        @test false
    end
end


module BalancedIncompleteBlockDesigns
using ChooseOptimizer
using JuMP, MathProgBase, Primes

export BIBD, projective, BIBD_check, blocks

"""
`BIBD(b,v,r,k,λ)` finds a `(b,v,r,k,λ)`-balanced incomplete block design.
The return is a `v`-by-`b` matrix of 0s and 1s whose i,j-entry  is 1
exactly when vertex i is in block j. Errors erupt if no such design exists.

The shorter form `BIBD(b,k,λ)` returns a symmetric design and is equivalent
to `BIBD(b,b,k,k,λ)`.
"""
function BIBD(b::Int,v::Int,r::Int,k::Int,l::Int)
    errmsg = "No ($b,$v,$r,$k,$l)-BIBD can be found"
    #set_solver(Gurobi)
    #set_solver_verbose(true)

    @assert b>0 && v>0 && r>0 && k>0 && l>0 && k<v "Invalid parameters: "*errmsg
    @assert b*k==v*r && r*(k-1)==l*(v-1) errmsg

    M = Model(get_solver())

    @variable(M, x[1:v,1:b], Bin)       # 1{vtx i in block B}
    @variable(M, y[1:v,1:v,1:b], Bin)   # 1{vtcs i,j in block B} where i≠j

    # Basics for y
    for i=1:v
        for B=1:b
            @constraint(M, y[i,i,B]==0)  # i≠j
        end
    end

    for i=1:v
        for j=1:v
            for B=1:b
                @constraint(M, y[i,j,B]==y[j,i,B]) # symmetric in i,j
            end
        end
    end

    # Each vertex is in r blocks
    for i=1:v
        @constraint(M, sum(x[i,B] for B=1:b) == r)
    end

    # Each block has k vertices
    for B=1:b
        @constraint(M, sum(x[i,B] for i=1:v) == k)
    end

    # every pair i≠j are in l blocks together
    for i=1:v
        for j=1:v
            if i!=j
                @constraint(M, sum(y[i,j,B] for B=1:b) == l)
            end
        end
    end

    # if i,j in B, then x[i,B] and x[j,B] must be 1
    for i=1:v
        for j=1:v
            for B=1:b
                @constraint(M, x[i,B] >= y[i,j,B])
            end
        end
    end

    # if i,j in B, force y[i,j,B] to 1
    for i=1:v
        for j=1:v
            if i!=j
                for B=1:b
                    @constraint(M, y[i,j,B] >= x[i,B] + x[j,B] - 1)
                end
            end
        end
    end

    optimize!(M)
    status = Int(termination_status(M))

    if status != 1
        error(errmsg)
    end

    X = Int.(value.(x))
end

BIBD(b::Int,k::Int,l::Int) = BIBD(b,b,k,k,l)

"""
`projective(n)` returns a projective plane of order `n` as a 0,1-matrix.
The rows of the matrix correspond to the points and the columns to the lines.
The i,j-entry is 1 iff point i is on line j.
"""
function projective(n::Int)
    @assert n>1 "Order of the plane must be at least 2"
    return BIBD(n*n+n+1,n+1,1)
end

function _all_same(vec::Array{T,1}) where T
    n = length(vec)
    if n < 2
        return true
    end
    for k=1:n-1
        if vec[k] != vec[k+1]
            return false
        end
    end
    return true
end

"""
`BIBD_check(A)` checks if the matrix `A` represents a
balanced incomplete block design. If so, it returns
its parameters as a 5-tuple `(b,v,r,k,λ)`.
"""
function BIBD_check(A::Array{Int,2})
    if maximum(A)>1 || minimum(A)<0
        error("Not a 0,1-matrix")
    end

    v,b = size(A)

    rlist = sum(A;dims=2)[:]
    if !_all_same(rlist)
        error("Replication number fail")
    end
    r = rlist[1]

    klist = sum(A;dims=1)[:]
    if !_all_same(klist)
        error("Block size fail")
    end

    k = klist[1]

    if k==v
        error("Incompleteness fail: v==k==$v")
    end

    B = A*A'
    l = B[1,2]
    for i=1:v
        for j=1:v
            if i!=j && B[i,j]!=l
                error("Lambda fail")
            end
        end
    end

    return (b,v,r,k,l)
end

"""
`blocks(A)` gives a list of the blocks of the balanced incomplete block design
specified by the matrix `A` (e.g., the output of `BIBD`)
"""
function blocks(A::Array{Int,2})
    b,v,r,k,l = BIBD_check(A)
    [ Set(findall(A[:,j] .> 0)) for j=1:b]
end

include("prime_plane.jl")

end  # end of module


#####################################################################################
#                                  Broomhead-King                                   #
#####################################################################################
import LinearAlgebra
export broomhead_king
"""
    broomhead_king(s::AbstractVector, d::Int) -> U, S, Vtr
Return the Broomhead-King coordinates of a timeseries `s`
by performing `svd` on high-dimensional embedding if `s` with dimension `d` with
minimum delay.

## Description
Broomhead and King coordinates is an approach proposed in [^Broomhead1987] that applies the
Karhunen–Loève theorem to delay coordinates embedding with smallest possible delay.

The function performs singular value decomposition
on the `d`-dimensional matrix ``X`` of ``s``,
```math
X = \\frac{1}{\\sqrt{N}}\\left(
\\begin{array}{cccc}
x_1 & x_2 & \\ldots & x_d \\\\
x_2 & x_3 & \\ldots & x_{d+1}\\\\
\\vdots & \\vdots & \\vdots & \\vdots \\\\
x_{N-d+1} & x_{N-d+2} &\\ldots & x_N
\\end{array}
\\right) = U\\cdot S \\cdot V^{tr}.
```
where ``x := s - \\bar{s}``.
The columns of ``U`` can then be used as a new coordinate system, and by
considering the values of the singular values ``S`` you can decide how many
columns of ``U`` are "important". See the documentation page for example application.

[^Broomhead1987]:  D. S. Broomhead, R. Jones and G. P. King, J. Phys. A **20**, 9, pp L563 (1987)
"""
function broomhead_king(x::AbstractArray, d::Int)
    X = trajectory_matrix(x, d)
    F = LinearAlgebra.svd(X)
    return F.U, F.S, F.Vt
end

function trajectory_matrix(x::AbstractArray, d::Int)
    xdash = mean(x)
    N = length(x); sqN = √N
    X = zeros(N-d+1, d)
    for j in 1:d
        for i in 0:(N-d)
            @inbounds X[i+1, j] = (x[j+i] - xdash)/sqN
        end
    end
    return X
end


function nlp_cvx_002_011(
    optimizer,
    objective_tol,
    primal_tol,
    dual_tol,
    termination_target = TERMINATION_TARGET_LOCAL,
    primal_target = PRIMAL_TARGET_LOCAL,
)
    model = Model(optimizer)

    @variable(model, x)
    @variable(model, y)

    @objective(model, Min, (x - 3)^2 + (y - 2)^2)
    @constraint(model, 1 * x - 3 * y <= 3)
    @constraint(model, 1 * x - 5 * y <= 0)
    @constraint(model, 3 * x + 5 * y >= 15)
    @constraint(model, 7 * x + 2 * y >= 20)
    @constraint(model, 9 * x + 1 * y >= 20)
    @constraint(model, 3 * x + 7 * y >= 17)

    optimize!(model)

    check_status(model, FEASIBLE_PROBLEM, termination_target, primal_target)
    check_objective(model, 0, tol = objective_tol)
    return check_solution([x, y], [3, 2], tol = primal_tol)
end


function PCA(X)
    mean_vec = mean(X, dims=2)
    X = X .- mean_vec
    M = X'*X
    e, EV = eigen(M)
    tmp = (X*EV)'
    V = tmp[end:-1:1,:]
    S = reverse(sqrt.(e))
    for i in 1:size(V, 2)
        V[:,i] ./= S
    end
    V, S, mean_vec
end

function get_projection(image_matrix, d)
    V, S, mean_vec = PCA(image_matrix)
    V'[:,1:d], mean_vec
end

function image_to_eigenface_rep(image, proj, mean_vec)
    proj'*(reshape(convert(Array{Float64}, img), :, 1) - mean_vec)
end

function eigenface_rep_to_image(eigenface_rep, proj, mean_vec, dims)
    image = proj*eigenface_rep + mean_vec
    Gray.(N0f8.(clamp01.(reshape(image, dims[1], dims[2]))))
end

function get_eigenfaces(proj, d, dims)
    eig(n) = rescale(proj[:,n])
    eigenfaces = reduce(hcat,[eig(i) for i in 1:d])
    image_matrix_to_images(eigenfaces, dims)
end


function rober!(du, u, p, t)
    y1, y2, y3 = u
    k1, k2, k3 = p

    du[1] = -k1*y1 + k3*y2*y3
    du[2] = k1*y1 - k2*y2^2. - k3*y2*y3
    du[3] = k2*y2^2.

end

rober_model = DEsteady(
    func = rober!,
    u0 = [1.0,0.0,0.0],
    p = [1.0,0.0,0.0]
)

function bistable_ode!(du, u, p ,t)
    s1, s2 = u
    K1, K2, k1, k2, k3, k4, n1 , n2  = p
    du[1] = k1 / (1 + (s2/K2)^n1) - k3*s1
    du[2] = k2/  (1 + (s1/K1)^n2) - k4*s2
end

bistable_model = DEsteady(
    func = bistable_ode!,
    p = [1., 1., 20., 20., 5., 5.,  4., 4.],
    u0 =  [3., 1.]
)


# A permutation is an ordered arrangement of objects. For example, 3124 is one
# possible permutation of the digits 1, 2, 3 and 4. If all of the permutations
# are listed numerically or alphabetically, we call it lexicographic order. The
# lexicographic permutations of 0, 1 and 2 are:
#
# 012   021   102   120   201   210
#
# What is the millionth lexicographic permutation of the digits:
# 0, 1, 2, 3, 4, 5, 6, 7, 8 and 9?

using ProjectEulerSolutions

# If the permutations are ordered, than we can calculate what percentage of the
# way 1,000,000 is through 10!.  We can extract the digit, find the remainder
# after subtracting from 1,000,000, then repeat for 9! then 8!, etc, extracting
# all the digits.
function p024solution(n::Integer=10)::String

    n = n - 1 # using one-indexing, not zero-indexing
    num_per_digit = factorial(10)

    digit_set = collect(0:9)

    perm = Vector{Integer}()
    for k = 0:9
        num_per_digit = fld(num_per_digit, length(digit_set))
        digit = digit_set[fld(n, num_per_digit)+1]
        push!(perm, digit)
        n -= num_per_digit * fld(n, num_per_digit)
        deleteat!(digit_set, findfirst(digit_set .== digit))
    end

    return mapreduce(x -> string(x), *, perm)
end

p024 = Problems.Problem(p024solution)

Problems.benchmark(p024, 1_000_000)


# Solução de Sistemas de Equações Lineares: Métodos iterativos
# Gauss-Jacobi e Gauss-Seidel
using LinearAlgebra, Plots, Printf

# Função para atualização do vetor x
###################################################################################################
cn_update_x(A, b, x, i) = (b[i] - A[i, :] ⋅ x + A[i, i] * x[i]) / A[i, i]
cn_gaussJacobi(A, b, x, x0, i) = cn_update_x(A, b, x0, i)
cn_gaussSeidel(A, b, x, x0, i) = cn_update_x(A, b, x, i)

# Função de erro
###################################################################################################
function error(x, x_)
    nx, nΔx = norm(x), norm(x - x_)
    if nx < eps(); return nΔx else; return nΔx / nx; end
end

# Função para solução de sistema de equações lineares por métodos iterativos
###################################################################################################
function cn_slIter(A, b, x0=zeros(length(b)); met=cn_gaussSeidel, tol=1e-8, maxit=50)
    # Matriz A deve ser quadrada
    n = length(b)
    @assert size(A) == (n, n)
    # Inicialização
    err, nite, x = tol+1, 0, copy(x0)
    x_hist, err_hist = zeros(n, maxit+1), zeros(maxit)
    x_hist[:, 1] = x
    # Procedimento iterativo
    while err > tol && nite < maxit
        nite += 1
        # Atualização dos valores das variáveis, x
        for i = 1:n; x[i] = met(A, b, x, x_hist[:, nite], i); end
        # Atualização do erro
        err = error(x, x_hist[:, nite])
        # Atualização dos históricos
        x_hist[:, nite+1], err_hist[nite] = x, err
    end
    # Retorna históricos com valores das variáveis e erros por iteração
    return x_hist[:, 1:nite+1], err_hist[1:nite]
end

# Função com os critérios de convergência
###################################################################################################
function cn_conv(A)
    abs_A, n = abs.(A), size(A, 1)
    # Critério das linhas e colunas
    row = vec(sum(abs_A, dims=2)) ./ diag(abs_A) .- 1
    col = vec(sum(abs_A, dims=1)) ./ diag(abs_A) .- 1
    # Critério de Sassenfeld
    sas = zeros(n); sas[1] = row[1]
    for i = 2:n
        sas[i] = (sum(abs_A[i, i+1:end]) + sas[1:i-1] ⋅ abs_A[i, 1:i-1]) / abs_A[i, i]
    end
    return maximum(row), maximum(col), maximum(sas)
end

# Exemplos
###################################################################################################
exemplo = 3
if exemplo == 1
    A = [3 -0.1 -0.2; 0.1 7 -0.3; 0.3 -0.2 10];
    b = [7.85, -19.3, 71.4]
    x0 = [0., 0, 0]; tol = 1e-4; maxit = 50
elseif exemplo == 2
    A = [2. 1; 1 -2]; b = [2., -2]
    x0 = [0., 0]; tol = 1e-2; maxit = 50
elseif exemplo == 3
    A = [10. 2 1; 1 5 1; 2 3 10]; b = [14., 11, 8]
    x0 = [0., 0, 0]; tol = 1e-2; maxit = 50
elseif exemplo == 4
    A = [10. 5 3 1; 0 10 1 -2; 4 8 2 -1; 2 1 1 2];
    b = [9., -5, 3, 12]
    x0 = [0., 0, 0, 0]; tol = 1e-5; maxit = 200
end

# Cálculo da solução e apresentação dos resultados
###################################################################################################
fmt(output) = round.(output, sigdigits=4)
println("\nA: ", fmt(A)); println("b: ", fmt(b))

row, col, sas = cn_conv(A)
println("\nCritérios de convergência:")
@printf("linha: %.2f, coluna: %.2f, Sassenfeld: %.2f\n", row, col, sas)

println("\n### Gauss-Jacobi ###")
xGJ_hist, errGJ_hist = cn_slIter(A, b, x0, met=cn_gaussJacobi, tol=tol, maxit=maxit)
println("$(length(errGJ_hist)) iterações")
println("x_GJ: ", fmt(xGJ_hist))
println("err_GJ: ", fmt(errGJ_hist))

println("\n### Gauss-Seidel ###")
xGS_hist, errGS_hist = cn_slIter(A, b, x0, met=cn_gaussSeidel, tol=tol, maxit=maxit)
println("$(length(errGS_hist)) iterações")
println("x_GS: ", fmt(xGS_hist))
println("err_GS: ", fmt(errGS_hist))

gr(title="Histórico de erros", xlabel="Iteração", ylabel="Erro", yaxis=:log)
plot(errGJ_hist, label="GJ");
plot!(errGS_hist, label="GS")


"""
    elimGaussWithPivoting(A,B)

adalah fungsi untuk menyelesaikan SPL `Ax=B` dengan `A` adalah matriks sembarang
dan `B` adalah vektor ruas kanan menggunakan metode eliminasi Gauss dengan pivoting.

# Examples
```jl
julia> A = [ 2 5 4 2 3
            4 1 0.4 6 5
            0.9 4 8 7 10
            5 7 0.5 2 7
            2 6 8 9 4];

julia> B = [26; 24.4; 32.8; 30.5; 53];

julia> Xn = elimGaussWithPivoting(A,B)
5×1 Array{Float64,2}:
  1.9999999999999993
  3.000000000000001
  1.0
  3.0
 -1.0000000000000007
```
return solusi SPL `X`

"""
function elimGaussWithPivoting(A,b)
    # Hitung ukuran matriks dan inisiasi solusi X
    n = length(b);
    X = zeros(n,1);
    # Buatlah matriks Gandeng [A|b]
    Aug = [A b];
    # Lakukan operasi baris dasar terhadap matriks gandeng
    for p = 1:n-1
        # Lakukan operasi pindah baris untuk menentukan nilai pivot
        val,j = findmax(abs.(Aug[p:n,p]));
        C = Aug[p,:];
        Aug[p,:] = Aug[j+p-1,:];
        Aug[j+p-1,:] = C;
        # Jika pivot bernilai nol, maka gagal.
        if Aug[p,p] == 0;
            error("Matriks tidak punya solusi tunggal")
        end
        # Lakukan eliminasi menggunakan operasi Eij(k)
        for i = p+1:n
            k = Aug[i,p]/Aug[p,p];
            Aug[i,:] = Aug[i,:] - k*Aug[p,:];
        end
    end
    # Pisahkan matriks gandeng, kemudian substitusi mundur.
    A = Aug[:,1:n];
    b = Aug[:,1+n];
    X = backsub(A,b);
end


# # [Spline interpolations](@id interpolation-example)
#
# ## Interpolating data
#
# BSplineKit can interpolate evenly and unevenly-distributed data.
#
# For example, we can try interpolating random data on randomly-distributed data
# points:

using Random
rng = MersenneTwister(42)

Ndata = 20
xs = range(0, 1; length = Ndata) .+ 0.01 .* randn(rng, Ndata)
sort!(xs)  # make sure coordinates are sorted
xs[begin] = 0; xs[end] = 1;   # not strictly necessary; just to set the data limits
ys = sinpi.(xs) .+ 0.02 .* randn(rng, Ndata);

# Let's start by plotting the generated data:

using CairoMakie
CairoMakie.activate!(type = "svg")  # hide
scatter(xs, ys; label = "Data", color = :black)

# To interpolate the data using splines, we may choose any arbitrary B-spline
# order ``k`` (in particular, ``k = 4`` corresponds to cubic splines).
# The main interpolation function is [`interpolate`](@ref).

using BSplineKit
S = interpolate(xs, ys, BSplineOrder(4))

# Let's plot the result:

lines!(0..1, x -> S(x); label = "k = 4", color = Cycled(4 - 3))
current_figure()  # hide

# We can also choose other interpolation orders for comparison:

for k ∈ (5, 6, 8)
    S = interpolate(xs, ys, BSplineOrder(k))
    lines!(0..1, x -> S(x); label = "k = $k", color = Cycled(k - 3))
end
axislegend()
current_figure()  # hide

# We see larger and larger oscillations, especially near the boundaries, as the
# spline order increases.
# We can try to fix this using **natural** splines, which impose some
# derivatives to be zero at the boundaries.

# ## Natural splines
#
# [Natural splines](https://en.wikipedia.org/wiki/Spline_(mathematics)#Examples)
# usually refer to cubic splines (order ``k = 4``) with the additional
# constraint ``S''(a) = S''(b) = 0`` at the boundaries (``x ∈ \{a, b\}``).
#
# In BSplineKit, this concept is generalised for all even spline orders ``k``,
# by setting all derivatives of order ``2, 3, …, k / 2`` to be zero at the
# boundaries.
# For instance, for ``k = 6`` (quintic splines), the constraint is ``S'' = S''' = 0``.
# We achieve this by using [basis recombination](@ref basis-recombination-api)
# to implicitly impose the wanted boundary conditions.
#
# The natural boundary condition not only allows to suppress oscillations near
# the boundaries, but is also quite convenient for interpolations, as it reduces
# the number of degrees of freedom such that the number of unique spline knots
# is equal to the number of data points.
# This simply means that one can set the knots to be equal to the data points.
# All of this is done internally when using this boundary condition.
#
# To impose natural boundary conditions, one just needs to pass
# [`Natural`](@ref) to `interpolate`, as illustrated below.

k = 8
S = interpolate(xs, ys, BSplineOrder(k))  # without BCs
Snat = interpolate(xs, ys, BSplineOrder(k), Natural())  # with natural BCs

# Let's look at the result:

scatter(xs, ys; label = "Data", color = :black)
lines!(0..1, x -> S(x); label = "k = $k (original)", linewidth = 2)
lines!(0..1, x -> Snat(x); label = "k = $k (natural)", linestyle = :dash, linewidth = 4)
axislegend()
current_figure()  # hide

# Clearly, the spurious oscillations are strongly suppressed near the
# boundaries.


import Pkg; Pkg.add(Pkg.PackageSpec(url="https://github.com/JuliaComputing/JuliaAcademyData.jl"))
using JuliaAcademyData; activate("Foundations of machine learning")

# ## Intro to neurons
#
# At this point, we know how to build models and have a computer automatically learn how to match the model to data. This is the core of how any machine learning method works.
#
# Now, let's narrow our focus and look at **neural networks**. Neural networks (or "neural nets", for short) are a specific choice of a **model**. It's a network made up of **neurons**; this, in turn, leads to the question, "what is a neuron?"

#-

# ### Models with multiple inputs
#
# So far, we have been using the sigmoid function as our model. One of the forms of the sigmoid function we've used is
#
# $$\sigma_{w, b}(x) = \frac{1}{1 + \exp(-wx + b)},$$
#
# where `x` and `w` are both single numbers. We have been using this function to model how the amount of the color green in an image (`x`) can be used to determine if an image shows an apple or a banana.
#
# But what if we had multiple data features we wanted to fit?
#
# We could then extend our model to include multiple features like
#
# $$\sigma_{\mathbf{w},b}(\mathbf{x}) = \frac{1}{1 + \exp(-w_1 x_1 - w_2 x_2 - \cdots - w_n x_n + b)}$$
#
# Note that now $\mathbf{x}$ and $\mathbf{w}$ are vectors with many components, rather than a single number.
#
# For example, $x_1$ might be the amount of the color green in an image, $x_2$ could be the height of the object in the picture, $x_3$ could be the width, and so forth. We can add information for as many features as we have! Our model now has more parameters, but the same idea of gradient descent ("rolling the ball downhill") will still work to train our model.
#
# This version of the sigmoid model that takes multiple inputs is an example of a **neuron**.

#-

# In the video, we see that one huge class of learning techniques is based around neurons, that is, *artificial neurons*. These are caricatures of real, biological neurons. Both *artificial* and *biological* neurons have several inputs $x_1, \ldots, x_n$, and a single output, $y$. Schematically they look like this:

include(datapath("scripts/draw_neural_net.jl"))

number_inputs, number_neurons = 4, 1

draw_network([number_inputs, number_neurons])

# We should read this as showing how information flows from left to right:
# - 4 pieces of input information arrive (shown in green on the left);
#
# - the neuron (shown in red on the right) receives all the inputs, processes them, and outputs a single number to the right.

#-

# In other words, a neuron is just a type of function that takes multiple inputs and returns a single output.
#
# The simplest interesting case that we will look at in this notebook is when there are just two pieces of input data:

draw_network([2, 1])

# Each link between circles above represents a **weight** $w$ that can be modified to allow the neuron to learn, so in this case the neuron has two weights, $w_1$ and $w_2$.
#
# The neuron also has a single bias $b$, and an **activation function**, which we will take to be the $\sigma$ sigmoidal function that we have been using. (Note that other activation functions can be used!)
#
# Let's call our neuron $f_{w_1,w_2, b}(x_1, x_2)$, where
#
# $$f_{w_1,w_2, b}(x_1, x_2) := \sigma(w_1 x_1 + w_2 x_2 + b).$$
#
# Note that $f_{w_1,w_2, b}(x_1, x_2)$ has 3 parameters: two weights and a bias.
#
# To simplify the notation, and to prepare for more complicated scenarios later, we put the two weights into a vector, and the two data values into another vector:
#
# $$
# \mathbf{w} = \begin{pmatrix} w_1 \\ w_2 \end{pmatrix};
# \qquad
# \mathbf{x} = \begin{pmatrix} x_1 \\ x_2 \end{pmatrix}.
# $$
#
# We thus have
#
# $$f_{\mathbf{w}, b}(\mathbf{x}) = \sigma(\mathbf{w} \cdot \mathbf{x} + b),$$
#
# where the dot product $\mathbf{w} \cdot \mathbf{x}$ is an abbreviated syntax for $w_1 x_1 + w_2 x_2$.

#-

# #### Exercise 1
#
# Declare the function `f(x, w, b)` in Julia. `f` should take vectors `x` and `w` as vectors and `b` as a scalar. Furthermore `f` should call
#
# ```julia
# σ(x) = 1 / (1 + exp(-x))
# ```
#
# What output do you get for
#
# ```julia
# f(3, 4, 5)
# ```
# ?

#-

# #### Solution

σ(x) = 1 / (1 + exp(-x))
f(x, w, b) = σ(w ⋅ x + b)   # use \cdot<TAB> to get "⋅"

## or alternatively, use juxtaposed multiplication with the transpose w'
## f(x, w, b) = σ(w'x + b)   # use \cdot<TAB> to get "⋅"

# **Tests**

@assert f(3, 4, 5) ≈ 0.99999996

Courses/Foundations of machine learning/1100.Intro-to-neurons.jl


### A Pluto.jl notebook ###
# v0.17.1

using Markdown
using InteractiveUtils

# ╔═╡ b77c944b-88e4-421d-a629-0d96bf95ea70
begin
	using Pkg
	cd(joinpath(dirname(@__FILE__),".."))
    Pkg.activate(pwd())
	# Simulation packages
	using Distributions, HypothesisTests
	# Plotting packages
	using StatsPlots, LaTeXStrings, Measures
end

# ╔═╡ 86a87f27-7b57-4420-8cef-836ac531a254
# Make cells wider
html"""<style>
/*              screen size more than:                     and  less than:                     */
@media screen and (max-width: 699px) { /* Tablet */
  /* Nest everything into here */
    main { /* Same as before */
        max-width: 1200px !important; /* Same as before */
        margin-right: 100px !important; /* Same as before */
    } /* Same as before*/

}

@media screen and (min-width: 700px) and (max-width: 1199px) { /* Laptop*/
  /* Nest everything into here */
    main { /* Same as before */
        max-width: 1200px !important; /* Same as before */
        margin-right: 100px !important; /* Same as before */
    } /* Same as before*/
}

@media screen and (min-width:1000px) and (max-width: 1920px) { /* Desktop */
  /* Nest everything into here */
    main { /* Same as before */
        max-width: 1000px !important; /* Same as before */
        margin-right: 100px !important; /* Same as before */
    } /* Same as before*/
}

@media screen and (min-width:1921px) { /* Stadium */
  /* Nest everything into here */
    main { /* Same as before */
        max-width: 1200px !important; /* Same as before */
        margin-right: 100px !important; /* Same as before */
    } /* Same as before*/
}
</style>
"""

# ╔═╡ bcbed56c-0c94-11eb-0e8c-2de8d1a20f47
md"""

# Simulation - a small intro

## General approach
When a simulation is used to answer a question or to solve a problem, the following steps are involved:
* Formulate the problem and plan the study
* Collect the data and formulate the simulation model
* Check the accuracy of the simulation model (assumptions, limitations etc.)
* Construct a computer program
* Test the validity of the simulation model
* Plan the simulations to be performed
* Conduct the simulation runs and analyze the results
* Present the conclusions

During this practical session, we will run small simulations going over all these steps.

"""

# ╔═╡ 349b8586-0c96-11eb-2e7b-5b3230d1cf27
md"""
## Casino game
A casino offers a coin flipping game with the following rules:
1. Each play of the game involves repeatedly flipping an unbiased coin
2. The game stops when the difference between the number of heads tossed and the number of tails is 3
3. If you decide to play, each flip costs you € 1. You are not allowed to quit during a play of the game.
4. You receive € 8 at the end of each play of the game.

The problem at hand is to decide whether to play or not and to get an idea of the expected gains and the length of each game. We might also look into the required simulation size and into a parametric study (e.g. influence of the required difference between heads/tails and/or the default gain).

*Tip:* it can be analytically proven that the true mean of the number of flips required for a play of this game is 9.

##### Steps in the simulation process

###### Formulate the problem and plan the study:
the problem is described in a clear way (cf. suppra). Given these rules, we want to find out
1. if it is a good idea to play
2. the value of the expected profit

###### Collect the data and formulate the simulation model:
Let $X$ be the outcome of one coin flip $\Rightarrow X\sim \mathcal{Be}(1/2)$

###### Check the accuracy of the simulation model:
Our model is complete, we make no assumptions

###### Construct a computer program
"""

# ╔═╡ 3a20552e-0c97-11eb-3f0d-2586b1bb46ed
begin
	"""
		game(limit::Int=3)

	Play a game for a given limit. Returns a dict with the scores and the game length
	"""
	function game(limit::Int=3)
		d = Dict("heads"=>0, "tails"=>0, "length"=>0)
		while abs(d["heads"] - d["tails"]) < limit
			d["length"] += 1
			if rand()<=1/2
				d["heads"]+=1
			else
				d["tails"]+=1
			end
		end

		return d
	end

	"""
		sim(N::Int=1000; limit::Int=3)

	run a simulation of the game.
	"""
	function sim(N::Int=1000; limit::Int=3, gain::Real=8)
		L = Array{Int64,1}()
		G = Array{Int64,1}()
		for _ in 1:N
			g = game(limit)
			push!(L,g["length"])
			push!(G,gain - g["length"])
		end

		return L,G
	end

	nothing
end

# ╔═╡ b1bae776-0dfe-11eb-291b-53f9a87c4a41
# other method of creating a game
begin
	mutable struct Game
		heads::Int
		tails::Int
		length::Int
	end

	function play!(g::Game, limit::Int=3)
		while abs(g.heads - g.tails) < limit
			g.length += 1
			if rand()<=1/2
				g.heads += 1
			else
				g.tails += 1
			end

		end

		return g
	end

	g = Game(0,0,0)
	play!(g)
end

# ╔═╡ 65c6476c-0c98-11eb-2c7d-3bc3ba8d51e2
md"""
###### Test the validity of the simulation model
A game should stop when the difference between the amount of heads and tails equals 3
```julia
begin
	g = game()
	@assert abs(g["heads"] - g["tails"]) == 3
end
```
"""

# ╔═╡ 5b08b538-0c97-11eb-0319-877b39c4c4ed
md"""
###### Plan the simulations to be performed
We will run a certain number of simulations. At this time we do not have additional information that we can use (optimal number to be determined later).
"""

# ╔═╡ 607e4392-0c98-11eb-0a9d-ddfab6b42b26
md"""
###### Conduct the simulation runs and analyze the results
"""

# ╔═╡ 75d35828-0c99-11eb-1888-e9e7be234e89
begin
	# get data for game length and gains
	L,G = sim()
end;

# ╔═╡ fae1e448-0c98-11eb-3721-dddb169e403f
# show results
plot(histogram(L,xlabel="Game length"),histogram(G,xlabel="Gains"),
     label="",grid=false,ylabel="Frequency",size=(600,250) ,bottom_margin=5mm)

# ╔═╡ a39a3a72-0c99-11eb-00df-e92eb7a6ce07
plot([boxplot(L,ylabel="Game length", ylims=(0,maximum(L)+4)),
	  boxplot(G,ylabel="Gains")]...,
     label="",grid=false,size=(300,250),xaxis=false)

# ╔═╡ ac87cb8a-0c9a-11eb-21d5-9981e408db29
md"""
#### Simulation length
Up to now we have worked with a simulation length 1000. We have no idea about the variation of each simulation result. We investigate this in order to find out how simulation length impacts our estimate.
"""

# ╔═╡ 1aa54e8a-0c9b-11eb-3b7c-f3be005991df
begin
	# evolution of the mean in function of the sample length
	x = Array{Int,1}()
	y = Array{Float64,1}()
	ntests = 30
	for N in 10 .^(1:5)
		for _ in 1:ntests
			(L,_) = sim(N,limit=3)
			push!(x,N)
			push!(y,mean(L))
		end
	end
end

# ╔═╡ 3236977a-0c9b-11eb-3875-2304eaec0a8e
begin
	scatter(x,y,xscale=:log10,
			xlabel="Number of games per simulation",
			ylabel="Average game length",title="",label="")
	plot!(x,9*ones(size(x)),color=:red,label="")
	title!("Towards optimal simulation length\n (30 datapoints per config)")
end

# ╔═╡ 7fafde70-0c9c-11eb-0403-9bfb6e42045b
md"""
#### Parametric study
suppose we consider a range of differences [2,4] and a range of gains [4, 12] where we play 1e5 games for each config.
"""

# ╔═╡ c8b107ba-0c9d-11eb-11c7-830d112ca82f
let
	Δ = 1:5
	Γ = 0:15
	N = Int(1e5)
	res = []
	i = 1
	for d in Δ
		for g in Γ
			_,G = sim(N; limit=d, gain=g)
			push!(res, mean(G))
		end
	end
	blup = reshape(res,:,length(Δ))
	p1 = heatmap(Δ, Γ, blup, xlabel="diff", ylabel="guaranteed gain", title="mean gain")
	p2 = heatmap(Δ, Γ, blup .>= 0, xlabel="diff", ylabel="guaranteed gain", title="positive gain")
	plot(p1,p2, size=(600, 300))
end

# ╔═╡ 0bfa3ffe-0c9a-11eb-2c99-3d8e57983c4c
md"""
###### Present the conclusions
We find that on average a game takes 9 turns, which will lead you to lose on average € 1. The game could be tempting given that in six out of ten cases the player will actually win (a small amount).
"""

# ╔═╡ 8df49b7e-0c9e-11eb-0048-9368c9d7bee9
md"""
## Project management
Suppose we want to simulate a industrial process that can be presented as the table shown below:

| Activity | Predecessors | Completion time |
|----------|--------------|---------------- |
| A | - | $\sim \mathcal{N}(6,1)$ |
| B | A | $\sim \mathcal{U}(6,10)$ |
| C | A | $\sim \mathcal{U}(1.5,2.5)$ |
| D | B,C | $\sim \mathcal{U}(1.5,3)$ |
| E | D | $\sim \mathcal{U}(3,6)$ |
| F | E | $\sim \mathcal{U}(2,5)$ |
| G | E | $\sim \mathcal{U}(3,5)$ |
| H | F,G | $\sim \mathcal{U}(4,7)$ |
| I | H | $\sim \mathcal{U}(5,7)$ |
| J | H | $\le 5$ |

We want to get an idea of:
1. the mean project completion time
2. the probability that the project will be finished in less than 36 time units
3. determine the critical subprocesses with a sensitivity chart (i.e. determine the impact of changing a specific subprocess on the completion time given all the others remain the same)

"""

# ╔═╡ 7aceebea-0d1d-11eb-05fa-d7374004b881
md"""
###### Formulate the problem and plan the study
The problem and the tasks are described above

###### Collect the data and formulate the simulation Model
Data model (coming from historical data) is known. The duration can be obtained from the a vector of duration times.

###### Construct a computer program
"""

# ╔═╡ 8f99cf60-0c9e-11eb-2b6d-b5ca5ed20a65
begin
	# completion time distributions
	D = [Normal(6,1);
		Uniform(6,10);
		Uniform(1.5,2.5);
		Uniform(1.5,4);
		Uniform(3,6);
		Uniform(2,5);
		Uniform(3,5);
		Uniform(4,7);
		Uniform(5,7);
		Uniform(0,5)]

	# central positions (for sensitivity analysis)
	p = [6; 8; 2; 2.25; 4.5; 3.5; 4; 5.5; 6; 2.5]
	"""
		 duration(t::Array)

	generate a random sample from an array of distributions for the production process
	"""
	function duration(t::Array)
		t[1] + max(t[2],t[3]) + t[4] + t[5] + max(t[6], t[7]) + t[8] + max(t[9], t[10])
	end

	"""
		PM(N::Int)

	Run a Project Management simulation N times that returns a vector of duration times
	"""
	function PM(D::Array, N::Int=100)
		# Generate data
		S = [rand.(D) for _ in 1:N]
		# get durations
		return map(duration, S)
	end

	nothing;
end

# ╔═╡ 864b050c-0d1e-11eb-03d8-8ffcf23e019e
md"""
###### Check the accuracy of the simulation model (assumptions, limitations etc.)
We use a normal distribution for the first activity, so it is theoretically possible that a negative value for A occurs, which does not make sense. In a similar way, using a uniform distribution sets hard limits for task durations.

You can make testcases to assert the correctness of the result calculated by the duration function.
"""

# ╔═╡ f9562612-0d1e-11eb-1e16-bfa53760c18c
begin
	v = [0 0 1 0 0 1 0 0 0 0]
	@assert duration(v) ==  2
end

# ╔═╡ d18a68b2-0d20-11eb-0f39-150f9bc9e0e9
md"""
###### Plan the simulations to be performed
We will generate a sample of a given length an look at the experimental CDF.
"""

# ╔═╡ da7801bc-0ca0-11eb-0480-33f07e16c85b
let
	# idea mean value
	N = 1000; x = 36;
	res = PM(D,N)
	avg_D = mean(res)
	#  CDF
	p1 = StatsPlots.cdensity(res,label="Experimental CDF",legend=:bottomright)
	title!("P(X <= $(x)): $(sum(res .< x)/N)")

	plot!([avg_D; avg_D], [0; 1], line=:dash, label="mean duration $(round(avg_D,digits=2))")
	ylabel!("F_{X}(x) ")
	xlabel!("duration")
	# PDF
	p2 = StatsPlots.density(res, label="Experimental PDF")
	xlabel!("duration")
	ylabel!("f_{X}(x)")
	# total plot
	plot(p1, p2)
end

# ╔═╡ 3d5f9648-0d21-11eb-14c4-65704b349ed6
md"""
###### Conduct the simulation runs and analyze the results
We have obtained the results. We can do some additional testing such as validating the normality of our simulation data.

The sensitivity analysis can be done by changing one-factor-at-a-time (OAT), to see what effect this produces on the output. I.e. we move one input variable, keeping others at their baseline (nominal) values and repeat this for each of the other inputs in the same way.
"""

# ╔═╡ d8b821a0-0d21-11eb-08bc-f7027b1cbea1
begin
	s = PM(D,1000)
	HypothesisTests.ExactOneSampleKSTest(s,Normal(mean(s),std(s)))
end

# ╔═╡ d89eb800-0d21-11eb-29d9-c119c9347dbc
begin
	"""
		sensitivity(D, p, i::Int, N::Int=1000)

	Generate a sample where all values keep their baseline values with the exception of component i
	"""
	function sensitivity(D, p, i, N::Int=1000)
		# generate data
		data = [[p[1:i-1]; rand(D[i]); p[i+1:end]] for _ in 1:N]
		# return durations
		map(duration,data)
	end

	blup = [sensitivity(D,p, i) for i in 1:10]
	StatsPlots.boxplot(blup, label="")
	xlabel!("process step")
	ylabel!("duration boxplot")
	ylims!(30,40)
	xlims!(0,11)
	xticks!(collect(1:10))
	title!("Sensitivity analysis")
end

# ╔═╡ d148f3ea-0d21-11eb-2d67-f1f4c77248d5
md"""
###### Present the conclusions
With respect to the initial goals, we can conclude the following:
* the mean project completion time is around 37 time units
* the probability that the project will be finished in less than 36 time units is around 34%
* processes 3 and 10 do not impact the total project duration. Steps two and four account for the largest variability.
"""

# ╔═╡ Cell order:
# ╟─86a87f27-7b57-4420-8cef-836ac531a254
# ╠═b77c944b-88e4-421d-a629-0d96bf95ea70
# ╟─bcbed56c-0c94-11eb-0e8c-2de8d1a20f47
# ╟─349b8586-0c96-11eb-2e7b-5b3230d1cf27
# ╠═3a20552e-0c97-11eb-3f0d-2586b1bb46ed
# ╠═b1bae776-0dfe-11eb-291b-53f9a87c4a41
# ╟─65c6476c-0c98-11eb-2c7d-3bc3ba8d51e2
# ╟─5b08b538-0c97-11eb-0319-877b39c4c4ed
# ╟─607e4392-0c98-11eb-0a9d-ddfab6b42b26
# ╠═75d35828-0c99-11eb-1888-e9e7be234e89
# ╠═fae1e448-0c98-11eb-3721-dddb169e403f
# ╠═a39a3a72-0c99-11eb-00df-e92eb7a6ce07
# ╟─ac87cb8a-0c9a-11eb-21d5-9981e408db29
# ╠═1aa54e8a-0c9b-11eb-3b7c-f3be005991df
# ╠═3236977a-0c9b-11eb-3875-2304eaec0a8e
# ╟─7fafde70-0c9c-11eb-0403-9bfb6e42045b
# ╠═c8b107ba-0c9d-11eb-11c7-830d112ca82f
# ╟─0bfa3ffe-0c9a-11eb-2c99-3d8e57983c4c
# ╟─8df49b7e-0c9e-11eb-0048-9368c9d7bee9
# ╟─7aceebea-0d1d-11eb-05fa-d7374004b881
# ╠═8f99cf60-0c9e-11eb-2b6d-b5ca5ed20a65
# ╟─864b050c-0d1e-11eb-03d8-8ffcf23e019e
# ╠═f9562612-0d1e-11eb-1e16-bfa53760c18c
# ╟─d18a68b2-0d20-11eb-0f39-150f9bc9e0e9
# ╠═da7801bc-0ca0-11eb-0480-33f07e16c85b
# ╟─3d5f9648-0d21-11eb-14c4-65704b349ed6
# ╠═d8b821a0-0d21-11eb-08bc-f7027b1cbea1
# ╠═d89eb800-0d21-11eb-29d9-c119c9347dbc
# ╟─d148f3ea-0d21-11eb-2d67-f1f4c77248d5


using StatisticalRethinking, DataFrames, NamedArrays, OrderedCollections

na = NamedArray(
    [18 134 152; 3 4 7; 21 138 159],
    (   [:vaccinated, :unvaccinated, :total],
        [:varicella, :noninfected, :total]
    ),
    ("Rows", "Cols")
)

na |> display
println()

# Risk of varicella among vaccinated children = 18 / 152 = 0.118 = 11.8%

risk_vac = na[:vaccinated, :varicella] / na[:vaccinated, :total]

# Risk of varicella among unvaccinated children = 3 / 7 = 0.429 = 42.9%

risk_unvac = na[:unvaccinated, :varicella] / na[:unvaccinated, :total]

# Risk ratio = 0.118 / 0.429 = 0.28

rr = risk_vac / risk_unvac

# EXAMPLE: Calculating Vaccine Effectiveness
# Calculate the vaccine effectiveness from the varicella data in Table 3.13.

# VE = (42.9 – 11.8) / 42.9 = 31.1 / 42.9 = 72%

ve = (risk_unvac - risk_vac) /risk_unvac

# Alternatively, VE = 1 – RR = 1 – 0.28 = 72%

1 - rr |> display

# So, the vaccinated group experienced 72% fewer varicella cases
# than they would have if they had not been vaccinated.

df_covid = DataFrame(
    :completed => [true, true, false, false],
    :company => [:pfizer, :moderna, :jj, :astra],
    :unvaccinated => [18261, 15000, missing, missing],
    :vaccinated => [18262, 15000, missing, missing],
    :inf_unv => [162, 185, missing, missing],
    :inf_vac => [8, 11, missing, missing]
)

df_covid |> display

df = df_covid[df_covid.completed, :]


# -*- coding: utf-8 -*-
# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .jl
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.4.2
#   kernelspec:
#     display_name: Julia 1.4.1
#     language: julia
#     name: julia-1.4
# ---

# # ヒストグラム平坦化

# # 準備

# ## Load Julia Packages

using Images
using TestImages
using Plots

# ## read testimage

c = testimage("c")

# ## Visualize gray scale image histogram

data=reshape(255 .* channelview(c),prod(size(c)))
histogram(data, bins=0:255,normalize=:pdf)

# ## Visualize cumulative distribution function (so called CDF)

# +
i2c = Dict{Int,Int}()
for i in 0:255
    i2c[i] = 0
end

for i in Int.(255 .* channelview(c))
    i2c[i] += 1
end
# -

arr = [i2c[i] for i in 0:255]
cdf = [sum(arr[begin:i]) for i in 1:length(arr)]
plot(cdf./length(c), label="CDF")

# # Histogram Equalization

# - $q$ を画像の輝度の分布を表す確率密度関数とする. ヒストグラムを平坦化することは分布が（なるべく)一様分布に近似されれば良い.つまり累積分布関数が $y=ax$ のように原点と積分範囲の右端(画像であれば255=256-1)で $y=1$ となる一次関数になっていれば良い.
# - 今述べたことを定式化する. $X$ を画像の分布を表す確率変数だとする. $Y$ をヒストグラム平坦化により得られら分布とし滑らかに $Y=f(X)$ と変化したとする. さらに変換が可逆だと仮定する. $p$ を $Y$ の確率密度函数とすれば, 次のことを要請する.
#
# $$
# \frac{y}{255} = \int_{0}^{y} p(\tilde{y})d\tilde{y}
# $$

# 一般に $Y=f(X)$ と変換された確率変数の確率密度関数 $p$ は
#
# $$
# p(y) = \int \delta(y-f(x))q(x)dx
# $$
#
# である. この一般論を上式の右辺に適用すれば $p(y) = \frac{q(x)}{f'(x)}\ \mathrm{where}\ y=f(x)$ が従う.
#
# $$
# \begin{aligned}
# \frac{1}{255} &= \frac{d}{dy}\left(\frac{y}{255}\right) \\
#               &= \frac{d}{dy}\int_{0}^{y} p(\tilde{y})d\tilde{y} \\
#               &= p(y) \\
#               &= \frac{q(x)}{f'(x)}\ \mathrm{where}\ y=f(x) \\
# \end{aligned}
# $$
#
# よって
#
# $$
# f'(x) = 255q(x)
# $$
#
# 要するに欲しい変換は累積分布函数に255(確率変数 $X$ がとる最大値) を乗じたものである.
#
# $$
# f(x) = 255 \int_{0}^x q(\tilde{x}) d\tilde{x}
# $$
#
# これを画像処理に適用することを考えると積分は離散和になる. $q(x)$ は輝度が$x\in [0, 255]$である確率になる. 平たくいうと $x$ での（正規化された）ヒストグラムの値である．これを踏まえて下記のように実装をしてみよう.

n_pixels = length(c)
function hist_eq(p)
    s = 0
    for i in 0:p
        s+=i2c[i]
    end
    floor(255*s/n_pixels)
end

new_c=hist_eq.(255 .* channelview(c))

colorview(Gray, new_c./255)

histogram(new_c[:], normalize=:pdf, bins=0:255)

# +
new_i2c = Dict{Int,Int}()
for i in 0:255
    new_i2c[i] = 0
end

for i in new_c
    new_i2c[i] += 1
end
# -

# # 結果の確認

arr = [new_i2c[i] for i in 0:255]
cdf = [sum(arr[begin:i]) for i in 1:length(arr)]
plot(cdf./length(c))
# 直線になっている様子がわかるはず


# 'cstrt_reinhardt_ngon' provides the vertices coordinates (a,b) of a
# Reinhardt (m*k)-gon
function cstrt_reinhardt_ngon(m,k)
if mod(m,2) == 1 && m>=3 && k>=1
    a = zeros(m*k-1); b = zeros(m*k-1);
    for i = 1:m-1
        a[i*k] = sin(2*pi*i/m)/(2*cos(pi/(2*m)));
        b[i*k] = (1 - cos(2*pi*i/m))/(2*cos(pi/(2*m)));
    end
    if k>1
        for j = 1:k-1
            a[Int(k*(m-1)/2+j)] = cos((1+(m-3)/2+j/k)*(pi/m));
            b[Int(k*(m-1)/2+j)] = sin((1+(m-3)/2+j/k)*(pi/m));
            for i = 1:Int((m-1)/2)
                a[Int((i+(m-1)/2)*k+j)] = a[i*k] + cos((2*i+1+(m-3)/2+j/k)*(pi/m));
                b[Int((i+(m-1)/2)*k+j)] = b[i*k] + sin((2*i+1+(m-3)/2+j/k)*(pi/m));
                a[Int(m*k-((i+(m-1)/2)*k+j))] = -a[Int((i+(m-1)/2)*k+j)];
                b[Int(m*k-((i+(m-1)/2)*k+j))] = b[Int((i+(m-1)/2)*k+j)];
            end
        end
    end
    return(a,b)
end
end


#=
P34 (**) Calculate Euler's totient function phi(m).
Euler's so-called totient function phi(m) is defined as the number of positive integers r (1 <= r < m) that are coprime to m.
Example: m = 10: r = 1,3,7,9; thus phi(m) = 4. Note the special case: phi(1) = 1.

* (totient-phi 10)
4

Find out what the value of phi(m) is if m is a prime number. Euler's totient function plays an important role in one of the most widely used public key cryptography methods (RSA). In this exercise you should use the most primitive method to calculate this function (there are smarter ways that we shall discuss later).
=#

# From P32 but more terse
function gcd(a, b)
    b == 0 ? a : gcd(b, mod(a, b))
end

# From P33
function coprime(a, b)
    gcd(a, b) == 1
end

function totient_phi(m)
    # Starting at 2, not 1 (special case)
    coprime_to_m = filter(r -> coprime(r, m), 2:m)
    length(coprime_to_m) + 1
end

@assert totient_phi(10) == 4

println("Tests passed: JL-34.jl")


using DifferentialEquations
using Plots

micro_1 = @ode_def Micro1 begin
    dS = r*(1-S/K)*S - β*S*I
    dI = β*S*I-(μ+α)*I
    end β r μ K α

w = 1;
m = 10;
β = 0.0247*m*w^0.44;
r = 0.6*w^-0.27;
μ = 0.4*w^-0.26;
K = 16.2*w^-0.7;
α = (m-1)*μ;

parms = [β,r,μ,K,α];
init = [K,1.];
tspan = (0.0,10.0);

sir_prob = ODEProblem(micro_1,init,tspan,parms)

sir_sol = solve(sir_prob);

# Plot
plot(sir_sol,xlabel="Time",ylabel="Number")

m = [5,10,20,40]
ws = 10.^linspace(-3,3,601)
βs = zeros(601,4)
for i = 1:4
    βs[:,i] = 0.0247*m[i]*ws.^0.44
end

# Plot
plot(ws,βs,xlabel="Weight",ylabel="\\beta_min", xscale=:log10,yscale=:log10, label=["m = 5" "m = 10" "m = 20" "m = 40"],lw=3)


using Plots
using LaTeXStrings

# Probability of v±dv
p(v;m=1.,R=8.3145,T=298.15) = m/(R*T) * v * exp(-m*v^2/(2R*T))

# Probability of v < x, given m and T
pcum(x;m = 1., T = 298.15, R = 8.3145) = -exp(-m*x^2/(2*R*T)) + 1

# Inverse Cumulative Probability of v < x, given m and T
pinv(x;m=1., T=298.15, R=8.3145) = sqrt(-(2*R*T/m)*log(1-x))

# Start plot
plot(layout=(1,2))

# Collect x
x = collect(0:0.1:200)

# Set y with m = 1, T = 298.15K, R = 8.3145 J/Kmol
y = pcum.(x)

# plot
plot!(x,y,label="",subplot=1)
plot!(xlabel=L"x / m s^{-1}",ylabel=L"p~(v<x)",subplot=1)

xl = 70
yl = pcum(xl)
plot!([-10,xl],[yl,yl],label="",color="black",linestyle=:dash,subplot=1)
plot!([xl,xl],[-10,yl],label="",color="black",linestyle=:dash,subplot=1)
scatter!([xl],[yl],label="",color="black",markerstyle=:circle,subplot=1)

annotate!(xl,-0.07,text(L"v~(p_x)",8),subplot=1)
annotate!(-12,yl,text(L"p_x",8),subplot=1)

plot!(xlim=[0,200],ylim=[0,1],subplot=1)

# Sample points from inverse of cumulative distribution
N = 20_000
x = rand(N)
histogram!(pinv.(x),alpha=0.3,label="",normalize=:probability,bins=1:2:200,subplot=2)

# Plot distribution
x = collect(0:2:200)
plot!(x,2*p.(x),label="",linewidth=2,subplot=2)

plot!(xlabel=L"v / ms^{-1}",ylabel=L"p~(v)dv",subplot=2)
plot!(framestyle=:box,subplot=2)

plot!(size=(800,400),framestyle=:box)

savefig("./vsample.pdf")


using Random, HCubature, QuadGK, Distributions, Statistics, LinearAlgebra

function test_t_integral_1D_gaussian()

    rg = MersenneTwister(1)

    # easy case: standard univeriate Gaussian

    σ = 3.1

    μ = 0.1

    g = Normal(μ, σ)

    x = randn(rg)

    f(t) = pdf(g, t*x)

    # numerical integral

    num1 = hquadrature(f, -50, 50)[1]

    num2 = quadgk(f, -50, 50)[1]

    # analytical solution

    ana = 1/sqrt(2π*σ*σ) * sqrt(π) / sqrt(x*x/(2*σ*σ))

    num1, num2, ana, abs(num1-ana)

end

function test_t_integral_2D_diagonal_gaussian(seed)

    rg = MersenneTwister(seed)

    D = 3

    Σ = Diagonal(rand(rg, D)*2)

    U = svd(Σ).U

    μ = randn(rg, D)

    x = randn(rg, D)

    num1 = numericalintegraloft(x, μ, Σ)

    ana = analyticalintegraloft(x, μ, Σ)

    num1, ana, abs(num1-ana)

end

function test_t_integral_2D_full_gaussian(seed=1)

    rg = MersenneTwister(seed)

    D = 3

    Σ = randn(rg,D, D)*0.5; Σ = Σ*Σ' + 0.5*I

    U = svd(Σ).U

    μ = randn(rg, D)

    x = randn(rg, D)

    num1 = numericalintegraloft(x, μ, Σ)

    ana = analyticalintegraloft(x, μ, Σ)

    num1, ana, abs(num1-ana)

end

function numericalintegraloft(x, μ, Σ)

    g = MvNormal(μ, Σ)

    hquadrature(t -> pdf(g, t*x), -50, 50)[1]

end

function analyticalintegraloft(x, μ, Σ)

    D = length(μ) ; @assert(size(Σ, 1) == size(Σ, 2) == D == length(x))

    # see https://en.wikipedia.org/wiki/Gaussian_integral#Generalizations
    a =  0.5*x'*(Σ\x)
    b =  x'*(Σ\μ)
    c = -0.5*μ'*(Σ\μ)

    1/sqrt((2π)^D*det(Σ)) * sqrt(π/a) * exp(b^2/(4*a)+c)

end


"""
forwardsub(L,b)

Solve the lower-triangular linear system with matrix `L` and
right-hand side vector `b`.
"""
function forwardsub(L,b)

n = size(L,1)
x = zeros(n)
x[1] = b[1]/L[1,1]
for i = 2:n
    s = sum( L[i,j]*x[j] for j=1:i-1 )
    x[i] = ( b[i] - s ) / L[i,i]
end

return x
end

"""
backsub(U,b)

Solve the upper-triangular linear system with matrix `U` and
right-hand side vector `b`.
"""
function backsub(U,b)

n = size(U,1)
x = zeros(n)
x[n] = b[n]/U[n,n]
for i = n-1:-1:1
    s = sum( U[i,j]*x[j] for j=i+1:n )
    x[i] = ( b[i] - s ) / U[i,i]
end

return x
end

"""
lufact(A)

Compute the LU factorization of square matrix `A`, returning the
factors.
"""
function lufact(A)

n = size(A,1)
L = diagm(0=>ones(n))  # ones on main diagonal, zeros elsewhere
U = float(copy(A))

# Gaussian elimination
for j = 1:n-1
  for i = j+1:n
    L[i,j] = U[i,j] / U[j,j]   # row multiplier
    U[i,j:n] -= L[i,j]*U[j,j:n]
  end
end

return L,triu(U)
end


module SpectralProjectioGradientTests

using OptimPack
using Test

# The Rosenbrock function writes:
#
#    f(x) = sum((1 .- x[1:2:end]).^2) + sum(c10*(x[2:2:end] - x[1:2:end].^2).^2)
#
# The following method returns the cost function f(x) and overwrites
# the values of gx by the gradient ∇f(x).
function rosenbrock_fg!(x::AbstractVector{T},
                        gx::AbstractVector{T}) where {T<:AbstractFloat}
    # Check standard indexing and compatibility of dimensions.
    @assert !Base.has_offset_axes(x, gx)
    @assert size(x) == size(gx)

    # A few constants.
    c1 = T(1)
    c2 = T(2)
    c10 = T(10)
    c200 = T(200)

    # Compute f(x) and store ∇f(x) in gx.
    x1 = x[1:2:end]
    x2 = x[2:2:end]
    t1 = c1 .- x1
    t2 = c10*(x2 - x1.*x1)
    g2 = c200*(x2 - x1.*x1)
    gx[1:2:end] = -c2*(x1 .* g2 + t1)
    gx[2:2:end] = g2
    return sum(t1.*t1) + sum(t2.*t2)
end

function rosenbrock_init!(x0::AbstractVector{T}) where {T<:AbstractFloat}
  x0[1:2:end] .= -1.2
  x0[2:2:end] .=  1.0
  return x0
end

# The following function overwrites `dst` with `src` projected on the positive
# hyperorthant.
function nonnegative_projector!(dst::AbstractArray{T,N},
                                src::AbstractArray{T,N}) where {T,N}
    @assert axes(dst) == axes(src)
    @inbounds for i in eachindex(dst, src)
        dst[i] = max(src[i], zero(T))
    end
    return dst
end

T = Float32
n = 20
x0 = Vector{T}(undef, n)
xexact = ones(T, n) # exact solution

res = spg2(rosenbrock_fg!, copyto!, rosenbrock_init!(x0), 10;
           maxit=500, maxfc=1000, printer=nothing, verb=true)
x1 = res.x
println("Result without bounds: ", sqrt(sum((x1 - xexact).^2)/sum(xexact.^2)))

println()
res = spg2(rosenbrock_fg!,  nonnegative_projector!, rosenbrock_init!(x0), 10;
           maxit=500, maxfc=1000, printer=nothing, verb=true)
x2 = res.x

println("Result without bounds: ", sqrt(sum((x2 - xexact).^2)/sum(xexact.^2)))

end # module


function simp13(f,a,b,h)
    t = collect(Float64,a:h:b)
    oddsum, evensum = 0,0 #initialize
    for  j in 2:2:(length(t) -1)
        oddsum += f(t[j])
    end
    for j in 3:2:(length(t) -1)
        evensum += f(t[j])
    end
    I = (b-a)* ( f(t[1]) + 4*oddsum + 2*evensum + f(t[end]) )/(3*(length(t)-1))
    return I
end

function simp38(f,a,b,h)
    t = collect(Float64,a:h:b)
    return (b-a)/8*( f(t[1]) + 3*f(t[2]) + 3*f(t[3]) + f(t[4]) )
end

function testsimp2(f,a,b,h)
    t = collect(Float64,a:h:b)
    if mod(length(t)-1,2) == 0 #i.e. we have an even number of intervals
        return simp13(f,a,b,h)
    else #i.e. we have an odd number of intervals
        I1 = simp13(f,a,t[end-3],h)
        I2 = simp38(f,t[end-3],b,h)
        return I1 + I2
    end
end

#This seems to be the fastest
function intsimp(f,a,b,h)
    t = collect(Float64,a:h:b)

    if mod(length(t)-1,2) == 0 #i.e. we have an even number of intervals
        oddsum, evensum = 0,0 #initialize
        for  j in 2:2:(length(t) -1)
            oddsum += f(t[j])
        end
        for j in 3:2:(length(t) -1)
            evensum += f(t[j])
        end
        I = (b-a)* ( f(t[1]) + 4*oddsum + 2*evensum + f(t[end]) )/(3*(length(t)-1))
        return I
    else #i.e. we have an odd number of intervals
        # First we need to calculate the simpson 1/3 up until... end-3
        # since we have N-3 points for simpson 1/3 and N-4 intervals
        oddsum, evensum = 0,0 #initialize
        for  j in 2:2:(length(t) - 4)
            oddsum += f(t[j])
        end
        for j in 3:2:(length(t) - 4)
            evensum += f(t[j])
        end
        I1 = (t[end-3]-a)* ( f(t[1]) + 4*oddsum + 2*evensum + f(t[end-3]) )/(3*(length(t)-4))

        I2 = (b-t[end-3])/8*( f(t[end-3]) + 3*f(t[end-2]) + 3*f(t[end-1]) + f(t[end]) )

        return I1 + I2

    end
end

# Integration whenever a pair of vectors is given as a function, x and y
function intsimp(x::Array{Float64,1},y::Array{Float64,1})
    h = abs( x[2] - x[1] )

    if mod(length(x)-1,2) == 0 #i.e. we have an even number of intervals
        oddsum, evensum = 0,0 #initialize
        for  j in 2:2:(length(x) -1)
            oddsum += y[j]
        end
        for j in 3:2:(length(x) -1)
            evensum += y[j]
        end
        I = (x[end]-x[1])* ( y[1] + 4*oddsum + 2*evensum + y[end] )/(3*(length(x)-1))
        return I
    else #i.e. we have an odd number of intervals
        # First we need to calculate the simpson 1/3 up until... end-3
        # since we have N-3 points for simpson 1/3 and N-4 intervals
        oddsum, evensum = 0,0 #initialize
        for  j in 2:2:(length(x) - 4)
            oddsum += y[j]
        end
        for j in 3:2:(length(x) - 4)
            evensum += y[j]
        end
        I1 = (x[end-3]-x[1])* ( y[1] + 4*oddsum + 2*evensum + y[end-3] )/(3*(length(x)-4))

        I2 = (x[end]-x[end-3])/8*( y[end-3] + 3*y[end-2] + 3*y[end-1] + y[end] )

        return I1 + I2

    end
end


# The Fibonacci sequence is defined by the recurrence relation:
#
# F_n = F_n−1 + F_n−2, where F_1 = 1 and F_2 = 1.
# Hence the first 12 terms will be:
#
# F_1 = 1
# F_2 = 1
# F_3 = 2
# F_4 = 3
# F_5 = 5
# F_6 = 8
# F_7 = 13
# F_8 = 21
# F_9 = 34
# F_10 = 55
# F_11 = 89
# F_12 = 144
# The 12th term, F_12, is the first term to contain three digits.
#
# What is the index of the first term in the Fibonacci sequence to contain 1000
# digits?

using ProjectEulerSolutions

# There's a pattern in the number of digits in Fibonacci numbers.  It appears
# to be a new digit every 5 numbers or so.  We use that to estimate the upper
# bound for how far to generate numbers.
#
# map(x -> ndigits(x), fibonacci_count(48))
# [1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 5,
#  6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 8, 8, 8, 8, 9, 9, 9, 9, 9, 10, 10, 10, 10, 10]
function p025solution(n_digits::Integer=1)::Integer

    fibs = fibonacci_bound(BigInt(10)^(n_digits-1))
    n = length(fibs) + 1

    if ndigits(fibs[end]) < n_digits
        return n + 1
    else
        return n
    end
end

p025 = Problems.Problem(p025solution)

Problems.benchmark(p025, 1000)


"""
    eigenvector_centrality(g)

Compute the eigenvector centrality for the graph `g`.

Eigenvector centrality computes the centrality for a node based on the
centrality of its neighbors. The eigenvector centrality for node `i` is
the \$i^{th}\$ element of \$\\mathbf{x}\$ in the equation
``
    \\mathbf{Ax} = λ \\mathbf{x}
``
where \$\\mathbf{A}\$ is the adjacency matrix of the graph `g`
with eigenvalue λ.

By virtue of the Perron–Frobenius theorem, there is a unique and positive
solution if λ is the largest eigenvalue associated with the
eigenvector of the adjacency matrix \$\\mathbf{A}\$.

### References

- Phillip Bonacich: Power and Centrality: A Family of Measures.
    American Journal of Sociology 92(5):1170–1182, 1986
    http://www.leonidzhukov.net/hse/2014/socialnetworks/papers/Bonacich-Centrality.pdf
- Mark E. J. Newman: Networks: An Introduction.
       Oxford University Press, USA, 2010, pp. 169.
"""
eigenvector_centrality(g::AbstractGraph) = abs.(vec(eigs(adjacency_matrix(g), nev=1)[2]))::Vector{Float64}


#===============================================================================

The minimum number of cubes to cover every visible face on a cuboid measuring 3 x 2 x 1 is twenty-two.

If we then add a second layer to this solid it would require forty-six cubes to
cover every visible face, the third layer would require seventy-eight cubes, and
the fourth layer would require one-hundred and eighteen cubes to cover every
visible face.

However, the first layer on a cuboid measuring 5 x 1 x 1 also requires
twenty-two cubes; similarly the first layer on cuboids measuring 5 x 3 x 1, 7 x
2 x 1, and 11 x 1 x 1 all contain forty-six cubes.

We shall define C(n) to represent the number of cuboids that contain n cubes in
one of its layers. So C(22) = 2, C(46) = 4, C(78) = 5, and C(118) = 8.

It turns out that 154 is the least value of n for which C(n) = 10.

Find the least value of n for which C(n) = 1000.

--------------------------------------------------------------------------------

Let's first compute the number of cube a layer is composed of given an initial
rectangular parallelepiped of sides a, b and c along 1x, 1y and 1z.

Let chose a slice of size 1 along the plane 1xy, and consider the successive
layers. The figure below represent the initial cut with 0 (here for a=3 and
b=2), and three successive layers grown around it:

        3333
       322223
      32111123
     3210000123
     3210000123
      32111123
       322223
        3333

If P=2*(a+b) is the perimeter, then we see that the number of cubes is:
    Layer 1: P
    Layer 2: P+4
    Layer 3: P+8
    => For the layer n, the number of cubes is P+4(n-1). This is valid for all
slices of length 1, so it comes that the number of cubes in the layer N for all
slices is c(P+4(n-1)).

Doing so, we have covered the faces that are parallel to 1xz and 1yz, but we
still have to cover the two faces parallel to 1xy. Looking again that the
figure, it comes that the number of Cube to cover that face is
    Layer 1: A = ab, the initial area of that face
    Layer 2: Layer 1 + size(border Layer 1) = A + P
    Layer 3: Layer 2 + size(border Layer 2) = A + P + P + 4
    Layer 4: Layer 3 + size(border Layer 3) = A + P + P + 4 + P + 8

=> Layer n = A + (n-1) P + 4 (1+n-2)(n-2)/2
           = A + (n-1) (P + 2 (n-2))

Of course there is a front face and a back face.

So, all put together, it comes that the number of cubes composing the layer n
is N(n) = c(P+4(n-1)) + 2ab + 2(n-1) (P + 2 (n-2)). Replacing P by 2(a+b) and
simplifying, it comes:

    N(n) = 2(ab+bc+ac) + 4(n-1)(a+b+c+n-2)

The formula is obviously symmetric in a, b and c.

===============================================================================#

# Let check it the formula for N(n)

include("Euler.jl")
using Base.Test

n_cubes(n, a, b, c) = 2*(a*b+b*c+a*c) + 4*(n-1)*(a+b+c+n-2)

@test n_cubes(1, 1, 2, 3) ==  22
@test n_cubes(2, 1, 2, 3) ==  46
@test n_cubes(3, 1, 2, 3) ==  78
@test n_cubes(4, 1, 2, 3) == 118

function euler(threshold::Int)

    for half_nc in Euler.naturals(3)
        # nc is the number of cubes in a layer; half_nc is just the half of it

        # Assuming 1 ≤ a ≤ b ≤ c, we have to determine the number of solutions
        # of the equation nc == n_cubes(n, a , b, c).

        nbr_sol = 0

        for n in Euler.naturals(1)
            3 + 2*(n^2-1) ≤ half_nc || break

            for a in Euler.naturals(1)
                3a^2 + 2*(n-1)*(3a+n-2) ≤ half_nc || break

                for b in Euler.naturals(a)
                    b*(2a+b) + 2(n-1)*(a + 2b + n - 2) ≤ half_nc || break

                    num_c = half_nc - a*b -2*(n-1)*(a+b+n-2)
                    den_c = a + b + 2(n-1)

                    if rem(num_c, den_c) == 0
                        nbr_sol += 1
                    end
                end
            end
        end

        if nbr_sol == threshold
            return 2*half_nc
        end
    end
end

@test euler(10) == 154

@time println("Euler 126: $(euler(1000))")


#Julia 1.4 Environment
using Pkg
pkg"add ForwardDiff; precompile;"
pkg"add Plots; precompile;"
pkg"add PyPlot; precompile;"
import ForwardDiff  # calcula derivadas usando automatic differentiation in forward mode
using LinearAlgebra #adiciona operações de algebra linear

pyplot()#usa PyPlot como backend engine
∇ = (h,x)->ForwardDiff.gradient(h,x) #gradiente de h(x)
Hessian = (h,x)->ForwardDiff.hessian(h,x) # hessian de h(x)
Df = (h,x)->ForwardDiff.derivative(h,x);  # primeira derivada de h'(x)
D_2f = (h,x)->ForwardDiff.derivative(z->ForwardDiff.derivative(h,z),x) # segunda derivada h"(x)

#função de busca linear
function linear_search(f::Function,x::Vector,d::Vector,method::Function)
    ϕ = t->f(x+t*d)
    α = method(f,x,d)
    return (ϕ(α),x+α*d)
end

function secao_aurea(f::Function,x::Vector,d::Vector;φ::Function = t->f(x+t*d),ρ = 1/2,ϵ = 1e-9)::Float64
    θ_1 = (3 - sqrt(5))/2.0
    θ_2 = (sqrt(5) - 1)/2.0
    #Obtenção intervalo
    a=0
    b=2*ρ
    s = b / 2
    while(φ(b)<φ(s))
        a = s
        s = b
        b *= 2
    end
    #Obtenção de t
    u = a + θ_1*(b - a)
    v = a + θ_2*(b - a)
    while(b - a > ϵ)
        if(φ(u)<φ(v))
            b = v
            v = u
            u = a + θ_1*(b - a)
        else
            a = u
            u = v
            v = a + θ_2*(b - a)
        end
    end
    return (u+v)/2
end

function secao_aurea(f::Function,x::Vector,d::Vector,(a,b)::Tuple{Float64,Float64};φ::Function = t->f(x+t*d),ϵ = 1e-9)::Float64
    θ_1 = (3 - sqrt(5))/2.0
    θ_2 = (sqrt(5) - 1)/2.0
    #Obtenção de t
    u = a + θ_1*(b - a)
    v = a + θ_2*(b - a)
    while(b - a > ϵ)
        if(φ(u)< φ(v))
            b = v
            v = u
            u = a + θ_1*(b - a)
        else
            a = u
            u = v
            v = a + θ_2*(b - a)
        end
    end
    return (u+v)/2
end

function newton(f::Function,x::Vector,d::Vector;φ::Function = y->f(x+y*d),t::Float64=0.25,ϵ = 1e-9,n_iter=10000)::Float64
   iter = 0
   while (abs(Df(φ,t)) > ϵ && iter < n_iter)
         t = t - (Df(φ,t)/D_2f(φ,t))
         iter = iter + 1
   end
   return t
end

function armijo(f::Function,x::Vector,d::Vector;φ::Function = y->f(x+y*d),t::Float64= 1.0,η::Float64 = 0.25)
    # f(x + td) > f(x) +t(∇f(x)⋅ d)
    while (φ(t)>φ(0)+ η*t*(∇(f,x)⋅d))
      t *=0.8
    end
    return t
end


# Collection of utilities for special functions (Bessel/Hankel)

"Derivative of Hankel function of first kind, with respect to argument (z)"
function diffhankelh1(n,z)
    if n!=0
        0.5*(hankelh1(-1 + n, z) - hankelh1(1 + n, z))
    else
        - hankelh1(1, z)
    end
end

"Derivative of Bessel function of first kind, with respect to argument (z)"
function diffbesselj(n,z)
    if n!=0
        0.5*(besselj(-1 + n, z) - besselj(1 + n, z))
    else
        - besselj(1, z)
    end
end

fresnels(z::T) where T = T((1+im)//4*(erf((1+im)//2*sqrt(T(π))*z) - im*erf((1-im)//2*sqrt(T(π))*z) ))

fresnelc(z::T) where T = T((1-im)//4*(erf((1+im)//2*sqrt(T(π))*z) + im*erf((1-im)//2*sqrt(T(π))*z) ))


export kaplanyorke_dim

"""
    kaplanyorke_dim(λs::AbstractVector)
Calculate the Kaplan-Yorke dimension, a.k.a. Lyapunov dimension[^Kaplan1970].

## Description
The Kaplan-Yorke dimension is simply the point where
`cumsum(λs)` becomes zero (interpolated):
```math
 D_{KY} = k + \\frac{\\sum_{i=1}^k \\lambda_i}{|\\lambda_{k+1}|},\\quad k = \\max_j \\left[ \\sum_{i=1}^j \\lambda_i > 0 \\right].
```

If the sum of the exponents never becomes negative the function
will return the length of the input vector.

Useful in combination with [`lyapunovspectrum`](@ref).

[^Kaplan1970]: J. Kaplan & J. Yorke, *Chaotic behavior of multidimensional difference equations*, Lecture Notes in Mathematics vol. **730**, Springer (1979)
"""
function kaplanyorke_dim(v::AbstractVector)
    issorted(v, rev = true) || throw(ArgumentError(
    "The lyapunov vector must be sorted from most positive to most negative"))

    s = cumsum(v); k = length(v)
    # Find k such that sum(λ_i for i in 1:k) is still possitive
    for i in eachindex(s)
        if s[i] < 0
            k = i-1
            break
        end
    end

    if k == 0
        return zero(v[1])
    elseif k < length(v)
        return k + s[k]/abs(v[k+1])
    else
        return typeof(v[1])(length(v))
    end
end


@testset "Wilcoxon rank sum exact test" begin
    x1 = [0.80, 0.83, 1.89, 1.04, 1.45, 1.38, 1.91, 1.64, 0.73, 1.46]
    x2 = [1.15, 0.88, 0.90, 0.74, 1.21]
    res_wtest = wilcox_test(x1, x2)
    test_show(res_wtest)
    @test res_wtest.statistic ≈ 35
    @test res_wtest.pval ≈ 0.254412254412254
end

@testset "Wilcoxon rank sum test with continuity correction" begin
    df_sleep = dataset("datasets", "sleep")
    res_wtest = wilcox_test(@formula(Extra ~ Group), df_sleep)
    test_show(res_wtest)
    @test res_wtest.statistic ≈ 25.5
    @test res_wtest.pval ≈ 0.0693275754336266
end

@testset "Wilcoxon signed rank sum exact test" begin
    x1 = [1.83,  0.50,  1.62,  2.48, 1.68, 1.88, 1.55, 3.06, 1.30]
    x2 = [0.878, 0.647, 0.598, 2.05, 1.06, 1.29, 1.06, 3.14, 1.29]
    res_wtest = wilcox_test(x1, x2, paired=true)
    test_show(res_wtest)
    @test res_wtest.statistic ≈ 40
    @test res_wtest.pval ≈ 0.0390625
end

@testset "Wilcoxon signed rank sum test with continuity correction" begin
    x1 = [-2.0, -3.2, 1.2, 5.8, 19.6, 19.9, 28.9, 31.1, 27.9, 21.9, 8.8, 7.5]
    x2 = [-1.8, -2.9, 0.9, 1.5, 11.5, 18.8, 23.9, 24.9, 23.8, 19.4, 8.8, 4.8]
    res_wtest = wilcox_test(x1, x2, paired=true)
    test_show(res_wtest)
    @test res_wtest.statistic ≈ 62
    @test res_wtest.pval ≈ 0.011278190116074
end


using Plots, Distributions, Statistics, LaTeXStrings, ProgressBars, StatsBase, JLD

function SSMC(f :: Function , R :: Real, r :: Real, Θ :: Real, θ ::Real , N :: Integer)
    ΔR= R - r
    Δθ= Θ - θ
    dist=[ ΔR .* rand(N) .+ r, Δθ .* rand(N) .+ θ ]
    f₍ₓ₎=[]
    for n in 1:N
        push!(f₍ₓ₎, f(getindex.(dist, n)...))
    end
    ∫f₍ₓ₎dx= mean(f₍ₓ₎)* ΔR * Δθ
    error= std(f₍ₓ₎)/sqrt(N)
    return ∫f₍ₓ₎dx, error
end

samples=10^4
ErrorList=[]
IntegralRes=[]
for i in ProgressBar(1:samples)
    I, E=SSMC((r, θ) -> r^3 * sin(θ) * cos(θ) * (3 + r * cos(θ)), 1, 0 , π, 0, 10^5)
    push!(IntegralRes, I)
    push!(ErrorList, E)
end
histogram(IntegralRes, dpi=600, ylabel="Probability Density", xlabel="I",
    title="The Distribution of I's Results")
savefig("C:\\Users\\Narges\\Documents\\GitHub\\computational_physics\\chapter7\\7.2\\Figs\\IHistogram.png")
histogram(ErrorList, dpi=600,xlabel="Probability Density", ylabel="Error",
    title="The Distribution of Errors")
savefig("C:\\Users\\Narges\\Documents\\GitHub\\computational_physics\\chapter7\\7.2\\Figs\\EHistogram.png")
save("../../chapter7/7.2/data.jld","Idata",IntegralRes, "Edata", ErrorList )


#-------------julia-fractal.jl-------------------------------------------------#
#
# Purpose: This file tests the creation of the julia fractal in julia for fun
#
#------------------------------------------------------------------------------#

# We will need a few libraries for visualization
using PyPlot

# Function to calculate and output julia fractal to image
function create_fractal(cutoff::Float64, step::Float64, c_val::Float64,
                        res::Int64, range::Float64)
    # Setting up the array to be used
    carr = Array{Float64}(res, res)

    # For now, let's just assume that the imaginary and real ranges are from
    #     -range < x < range
    for i = 1:res
        for j = 1:res
            z = complex((i/res)*range - range/2.0,(j/res)*range - range/2.0)
            c = complex(0,c_val)
            n = 255
            while (abs(z) < cutoff && n > 5)
                z = z*z + c
                n -= 5
            end
            carr[i,j] = n/255.0;
        end
    end

    return carr

end

# Fuction to scan through C values
function c_scan(cutoff::Float64, step::Float64, res::Int64,
                range::Float64, max_c::Float64, c_step::Float64)

    id = 0
    for i = 0:c_step:max_c
        println(i)
        carr = create_fractal(cutoff, step, i, res, range)
        figure(figsize = (10,10))
        imshow(carr, cmap=ColorMap("Blues"))
        subplots_adjust(top = 1, bottom = 0, right = 1, left = 0,
                        hspace = 0, wspace = 0)
        savefig(string("c_scan", lpad(id, 5, 0), ".png"))
        id += 1
    end
end

# Function to zoom in on fractal
function fractal_zoom(cutoff::Float64, step::Float64, c_val::Float64,
                      res::Int64, max_range::Float64,
                      min_range::Float64)
    id = 0
    range = max_range
    while range > min_range
        println(range)
        carr = create_fractal(cutoff, step, c_val, res, range)
        figure(figsize = (10,10))
        imshow(carr, cmap=ColorMap("Blues"))
        subplots_adjust(top = 1, bottom = 0, right = 1, left = 0,
                        hspace = 0, wspace = 0)
        savefig(string("fractal_zoom", lpad(id, 5, 0), ".png"))
        id += 1
        range -= range*0.05
    end
end

fractal_zoom(10.0, 0.1, 1.0, 1024, 1.0, 0.0001)

#c_scan(10.,5., 1024, 3.0, 1.0, 0.025)


using TailRiskScenGen
using Distributions
## using Gadfly

mu = [0.0, 0.5]
cov = [[1 0.5], [0.5 1]]
num_scen = 1000

normal = MvNormal(mu, cov)
scenarios = rand(normal, num_scen)
probs = fill(1.0/num_scen, num_scen)

# print(scenarios)

new_scen, new_probs = aggregate_scenarios(scenarios, probs, mu, cov, eye(2), quantile(Normal(), 0.95))
# print(new_scenarios)

## x_min, y_min = minimum(scenarios,2)
## x_max, y_max = maximum(scenarios,2)
## x_scale = Scale.x_continuous(minvalue=x_min, maxvalue=x_max)
## y_scale = Scale.y_continuous(minvalue=y_min, maxvalue=y_max)
## p1 = plot(x=scenarios[1,:], y=scenarios[2,:], x_scale, y_scale, Guide.title("Full Scenario Set"))
## p2 = plot(x=new_scen[1,:], y=new_scen[2,:], x_scale, y_scale, Guide.title("Aggregated Scenario Set"))
## draw(PDF("data.pdf", 6inch, 6inch), vstack(p1,p2))


# illustrates effects of collinearity
# try setting rho=0.1 or rho=0.9, and observe
# the effect on standard dev of coefficients
using Econometrics, Plots, LinearAlgebra
function wrapper(rho, n)
  V = [1.0 rho; rho 1.0]
  P = cholesky(V).U
  x = randn(n,2)*P
  x = [ones(n,1) x]
  b = ones(3)
  y = x*b + randn(n,1)
  b1 = x\y

  z = x[:,1:2]
  b2 = z\y

  R = [0.0 1.0 -1.0]
  r = 0.0
  b3 = ols(y, x, R=R, r=r, silent=true)[1]
  results = [b1' b2' b3']
end

function main()
reps = 1000
rho = 0.9
n = 30
data = zeros(reps,8)
for rep = 1:reps
    data[rep,:] = wrapper(rho,n)
end
println("correlation between x2 and x3: ",rho)
println("descriptive statistics for 1000 OLS replications")
dstats(data[:,1:3])
println("descriptive statistics for 1000 OLS replications, dropping x3")
dstats(data[:,4:5])
println("descriptive statistics for 1000 Restricted OLS replications, b2=b3")
dstats(data[:,6:8])
p1 = histogram(data[:,2],title="OLS")
p2 = histogram(data[:,5],title="Drop X3")
p3 = histogram(data[:,7],title="Restricted LS")
plot(p1,p2,p3, layout=(3,1), show=true, legend=false)
gui()
savefig("collin.png")
return
end
main()