module OneDimensional

using DataFrames, Gadfly

export goldenSectionSearch, fibSearch

function ternarySearch(f, left, right, ɛ)
  callCnt, ls, rs, rels = 0, [left], [right], [1.0]

  while right - left > ɛ
    a = (left * 2 + right) / 3
    b = (left + right * 2) / 3
    if f(a) < f(b)
      right = b
    else
      left = a
    end
    push!(rels, (right - left) / (rs[end] - ls[end]))
    push!(ls, left)
    push!(rs, right)
    callCnt += 2
  end
  (left + right) / 2.0, callCnt, ls, rs, rels
end

function dichotomySearch(f, left, right, ɛ)
  callCnt, ls, rs, rels = 0, [left], [right], [1.0]

  δ = ɛ / 4
  while right - left > ɛ
    a = (left + right - δ) / 2
    b = (left + right + δ) / 2
    if f(a) < f(b)
      right = b
    else
      left = a
    end
    push!(rels, (right - left) / (rs[end] - ls[end]))
    push!(ls, left)
    push!(rs, right)
    callCnt += 2
  end
  (left + right) / 2.0, callCnt, ls, rs, rels
end

function goldenSectionSearch(f, left, right, ɛ)
  callCnt, ls, rs, rels = 2, [left], [right], [1.0]

  resphi = 2 - φ
  a = left + resphi * (right - left)
  b = right - resphi * (right - left)
  fa, fb = f(a), f(b)

  while right - left > ɛ
    if fa < fb
      right = b
      b = a
      fb = fa
      a = left + resphi * (right - left)
      fa = f(a)
    else
      left = a
      a = b
      fa = fb
      b = right - resphi * (right - left)
      fb = f(b)
    end
    push!(rels, (right - left) / (rs[end] - ls[end]))
    push!(ls, left)
    push!(rs, right)
    callCnt += 1
  end
  (left + right) / 2, callCnt, ls, rs, rels
end

function fibSearch(f, left, right, ɛ)
  callCnt, ls, rs, rels = 2, [left], [right], [1.0]

  fibs = [1, 2]
  while (right - left) / fibs[end] > ɛ
    push!(fibs, fibs[end] + fibs[end - 1])
  end

  x1 = left + (right - left) * fibs[end - 2] / fibs[end]
  #x2 = left + (right - left) * fibs[end - 1] / fibs[end]
  x2 = left + right - x1
  f1, f2 = f(x1), f(x2)

  for k=1:length(fibs)-3
    if f1 < f2
      right = x2
      x2 = x1
      #x1 = left + (right - left) * fibs[end - k - 2] / fibs[end - k]
      x1 = left + right - x2
      f2 = f1
      f1 = f(x1)
    else
      left = x1
      x1 = x2
      #x2 = left + (right - left) * fibs[end - k - 1] / fibs[end - k]
      x2 = left + right - x1
      f1 = f2
      f2 = f(x2)
    end
    push!(rels, (right - left) / (rs[end] - ls[end]))
    push!(ls, left)
    push!(rs, right)
    callCnt += 1
  end
  (left + right) / 2, callCnt, ls, rs, rels
end

function uniformSearch(f, left::Float64, right::Float64, n::Int32)
  fs = [f(left + i * (right - left) / (n + 1)) for i in 0:n+1]
  minval, j, = fs[1], 0
  for i in 2:length(fs)
    if minval > fs[i]
      minval, j = fs[i], i - 1
    end
  end
  left + j * (right - left) / (n + 1), minval, n + 1
end

function uniformSearch(f, left::Float64, right::Float64, ɛ::Float64)
  n = convert(Int32, (right - left) / ɛ - 1.0)
  return uniformSearch(f, left, right, n)
end

function uniformSequenceSearch(f, left, right, n::Int64, ɛ)
  callCnt, ls, rs, rels = 0, [left], [right], [1.0]

  while right - left > ɛ
    fs = [f(left + i * (right - left) / n) for i in 0:n]
    minval, j = fs[1], 1
    for i in 2:length(fs)
      if minval > fs[i]
        minval, j = fs[i], i - 1
      end
    end

    if j == 0
      right = left + (right - left) / n
    elseif j == length(fs) - 1
      left = right - (right - left) / n
    else
      a = left + (j - 1) * (right - left) / n
      b = left + (j + 1) * (right - left) / n
      left, right = a, b
    end

    push!(rels, (right - left) / (rs[end] - ls[end]))
    push!(ls, left)
    push!(rs, right)
    callCnt += n + 1
  end
  (left + right) / 2, callCnt, ls, rs, rels
end

type Line
  a :: Float64
  b :: Float64
  c :: Float64
end

function polylineSearch(f, L, left, right, ɛ)
  lineLipshitz(L, fx0, x0) = Line(-L, 1, L * x0 - fx0)

  function segmentLine(x1, y1, x2, y2)
    a = y1 - y2
    b = x2 - x1
    c = -a * x1 - b * y1
    Line(a, b, c)
  end

  function lineIntersection(l1::Line, l2::Line)
    Δ = det([[l1.a, l1.b] [l2.a, l2.b]])
    det([[-l1.c, l1.b] [-l2.c, l2.b]]) / Δ, det([[l1.a, -l1.c] [l2.a, -l2.c]]) / Δ
  end

  function minDiff(yc, fs)
    miny, j = yc[1], 1
    for i in 2:length(yc)
      if miny > yc[i]
        miny, j = yc[i], i
      end
    end
    fs[j] - yc[j]
  end

  fl, fr = f(left), f(right)
  pnt = lineIntersection(lineLipshitz(-L, fl, left), lineLipshitz(L, fr, right))
  xc = [left, pnt[1], right]
  yc = [fl, pnt[2], fr]
  fs = [fl, f(xc[2]), fr]
  callCnt = 3

  while minDiff(yc, fs) > ɛ
    # finding minimum value through polyline
    minval, j = yc[1], 1
    for i = 2:length(yc)
      if minval > yc[i]
        minval = yc[i]
        j = i
      end
    end

    # create two tangents
    newPike = f(xc[j])
    l1, l2 = lineLipshitz(-L, newPike, xc[j]), lineLipshitz(L, newPike, xc[j])
    callCnt += 1

    # cnage polyline insertint two new segments
    pnt1 = lineIntersection(l2, segmentLine(xc[j], yc[j], xc[j - 1], yc[j - 1]))
    pnt2 = lineIntersection(l1, segmentLine(xc[j], yc[j], xc[j + 1], yc[j + 1]))

    xc = [xc[1:j - 1]; pnt1[1]; xc[j]; pnt2[1]; xc[j + 1:end]]
    yc = [yc[1:j - 1]; pnt1[2]; newPike; pnt2[2]; yc[j + 1:end]]
    fs = [fs[1:j - 1]; f(pnt1[1]); newPike; f(pnt2[1]); xc[j + 1:end]]
    callCnt += 2
  end
  xc, yc, callCnt
end

minx, callNum, ls, rs, rels = ternarySearch(x -> x^2 + 2x, -5.0, 5.0, 0.001)
#df = DataFrame(L=ls, R=rs, Rel=rels)
#writetable("ternary.txt", df)
minx, callNum, ls, rs, rels = dichotomySearch(x -> x^2 + 2x, -5.0, 5.0, 0.001)
#df = DataFrame(L=ls, R=rs, Rel=rels)
#writetable("dichotomy.txt", df)
minx, callNum, ls, rs, rels = goldenSectionSearch(x -> x^2 + 2x, -5.0, 5.0, 0.001)
#df = DataFrame(L=ls, R=rs, Rel=rels)
#writetable("goldenSection.txt", df)
minx, callNum, ls, rs, rels = fibSearch(x -> x^2 + 2x, -5.0, 5.0, 0.001)
#df = DataFrame(L=ls, R=rs, Rel=rels)
#writetable("fib.txt", df)
#minf, fval = uniformSearch(x -> x^2 + 2x, -5.0, 5.0, 0.1)
minx, callNum, ls, rs, rels = uniformSequenceSearch(x -> x^2 + 2x, -5.0, 5.0, 5, 0.001)
#df = DataFrame(L=ls, R=rs, Rel=rels)
#writetable("uniform.txt", df)

eps = [0.1, 0.01, 0.001, 0.0001, 0.00001, 0.000000001]
ds, gs, fs, us = Int32[], Int32[], Int32[], Int32[]
f(x) = x^2 + 2x
for ɛ in eps
  _, dscnt, _, _, _ = dichotomySearch(f, -5.0, 5.0, ɛ)
  _, gscnt, _, _, _ = goldenSectionSearch(f, -5.0, 5.0, ɛ)
  _, fscnt, _, _, _ = fibSearch(f, -5.0, 5.0, ɛ)
  _, uscnt, _, _, _ = uniformSequenceSearch(f, -5.0, 5.0, 5, ɛ)
  push!(ds, dscnt)
  push!(gs, gscnt)
  push!(fs, fscnt)
  push!(us, uscnt)
end

dsit = [calls / 2.0 for calls in ds]
gsit = [calls - 2 for calls in gs]
fsit = [calls - 2 for calls in fs]
usit = [calls / 6 for calls in fs]

plot(layer(x=dsit, y=ds, Theme(default_color=color("purple")), Geom.point, Geom.line),
     layer(x=gsit, y=gs, Theme(default_color=color("orange")), Geom.point, Geom.line),
     layer(x=fsit, y=fs, Theme(default_color=color("green")), Geom.point, Geom.line),
     layer(x=usit, y=us, Theme(default_color=color("red")), Geom.point, Geom.line),
     Guide.xticks(ticks=[0:5:60]), Guide.yticks(ticks=[0:10:150]),
     Guide.title("Связь числа итераций и числа вычислений функции"),
     Guide.xlabel("Число итераций"), Guide.ylabel("Число вызовов функций"))

epsLogs = [-log(ɛ) for ɛ in eps]

plot(layer(x=epsLogs, y=ds, Theme(default_color=color("purple")), Geom.point, Geom.line),
     layer(x=epsLogs, y=gs, Theme(default_color=color("orange")), Geom.point, Geom.line),
     layer(x=epsLogs, y=fs, Theme(default_color=color("green")), Geom.point, Geom.line),
     layer(x=epsLogs, y=us, Theme(default_color=color("red")), Geom.point, Geom.line),
     Guide.title("Зависимость числа вызовов функции от минус логарифма точности"),
     Guide.xlabel("Точность"), Guide.ylabel("Число вызовов функций"))

xlines, ylines, cntCall = polylineSearch(x -> 0.5 * x^2 * sin(2x), 6, -3.0, 3.0, 0.5)

Gadfly.set_default_plot_size(12cm, 12cm)
plot(layer(x -> 0.5 * x^2 * sin(2x), -3, 3),
     layer(x=xlines, y=ylines, Theme(default_color=color("orange")), Geom.point, Geom.line),
     Guide.xticks(ticks=[-3:1:3]),
     Guide.title("Метод ломанныx функции y=1/2 ⋅ x^2 ⋅ sin(2x)"),
     Theme(default_color=color("purple")))

end


function counting_sort(A, k)
    B = Array(zeros(Int, (length(A),))')
    C = zeros(Int, (k+1,))

    for j in 1:length(A)
        C[A[j]+1] += 1
    end # C[i+1] now contains number of elems equal to i

    for i in 1:k
        C[i+1] += C[i]
    end # C[i+1] now contains number of elems ≤ i

    for j in range(length(A), 1, step = -1)
        B[C[A[j]+1]] = A[j]
        C[A[j]+1] -= 1
    end # B is sorted version of A

    return B
end

function radix_sort!(A)
    max = maximum(A)

    place = 1
    while max // place > 0
        counting_sort!(A, place)
        place *= 10
    end
end

function counting_sort!(A, place)
    B = Array(zeros(Int, (length(A),))')
    C = zeros(Int, (10,))

    for j in 1:length(A)
        index = Int(floor(A[j] / place))
        digit = (index % 10) + 1
        C[digit] += 1
    end # C[i+1] now contains number of elems equal to i

    for i in 1:9
        C[i+1] += C[i]
    end # C[i+1] now contains number of elems ≤ i

    for j in range(length(A), 1, step = -1)
        index = Int(floor(A[j] / place))
        digit = (index % 10) + 1
        B[C[digit]] = A[j]
        C[digit] -= 1
    end # B is sorted version of A

    # copy
    A .= B
end


function collatz_steps(t::Int64)
    t < 1 && throw(DomainError(t))
    s::Array = []
    while true
        if t % 2 == 0
            t = t ÷ 2
        else
            t = 3t + 1
        end
        push!(s,t)
        t == 1 && break
    end
    return s
end

function collatz_steps_02(t::Int64)
    t < 1 && throw(DomainError(t))
    s = []
    collatz = (x) -> iseven(x) ? x ÷ 2 : 3x + 1
    while true
        t = collatz(t)
        push!(s,t)
        t == 1 && break
    end
    return s
end

function max_stop_time(t::Int64)
    t < 1 && throw(DomainError(t))
    l = []
    w = []
    for i in 1:t
        push!(l, collatz_steps(i))
    end
    for j in l
        push!(w, length(j))
    end
    return findmax(w)
 end

function max_stop_time_02(t::Int64)
    t < 1 && throw(DomainError(t))
    l::Tuple{Int, Array} = (0,[])
    Threads.@threads for i in 1:t
        m = collatz_steps(i)
        if length(m) > length(l[2])
             l = (i,m)
        end
    end
    return l[1],length(l[2])
end

function max_stop_time_03(t::Int64)::Tuple{Int64,Int64}
    collatz = (x::Int64) -> iseven(x) ? x ÷ 2 : 3x + 1
    eval = (s) ->
    begin
        x = 0
        while s > 1
            s = collatz(s)
            x += 1
        end
        x
    end
    m::Tuple{Int64,Int64} = (0,0)
    Threads.@threads for i in 1:t
        if (s = eval(i)) > m[2] m = (i,s) end
    end
    return m
end


#=
   https://www.allendowney.com/blog/2018/10/21/the-game-of-ur-problem/
   """
   Here’s a probability puzzle to ruin your week.

   In the Royal Game of Ur, players advance tokens along a track with 14 spaces.
   To determine how many spaces to advance, a player rolls 4 dice with 4 sides. Two corners
   on each die are marked; the other two are not. The total number of marked corners —
   which is 0, 1, 2, 3, or 4 — is the number of spaces to advance.

   For example, if the total on your first roll is 2, you could advance a token to space 2.
   If you roll a 3 on the next roll, you could advance the same token to space 5.

   Suppose you have a token on space 13. How many rolls did it take to get there?
   """

   See:
   https://www.allendowney.com/blog/lions-and-tigers-and-bears/

   Allen Downey's solution:
   http://nbviewer.jupyter.org/github/AllenDowney/ThinkBayes2/blob/master/solutions/game_of_ur_soln.ipynb?flush=true

   cf ~/blog/game_of_ur_problem.blog
      ~/webppl/game_of_ur_problem.wppl

=#

using Turing, StatsPlots, DataFrames
include("jl_utils.jl")

@model function game_of_ur_problem2()

    function roll1(a)
        ss = sum(a)
        if ss == 13
            return  a
        elseif ss > 13
            return []
        else
            t = rand(DiscreteUniform(0,4))
            return roll1(vcat(a,t))
        end
    end

    # Simpler
    function roll(s,len)
        if s == 13
            return  len
        elseif s > 13
            return 0
        else
            t = rand(DiscreteUniform(0,4))
            return roll(s+t,len+1)
        end
    end

    len ~ Dirac(roll(0,0))
    # Skip length 0 events
    true ~ Dirac(len > 0)

end

model = game_of_ur_problem2()

num_chns = 4

# chns = sample(model, Prior(), MCMCThreads(), 10_000, num_chns)

# chns = sample(model, MH(), 10_000)
# chns = sample(model, PG(20), 1_000)
chns = sample(model, SMC(1000), 40_000)
# chns = sample(model, IS(), 10_000)

display(chns)
# display(plot(chns))

println("Distribution of len")
show_var_dist_pct(chns, :len)


import Base: in, isequal, isempty, length, union

#####
##### Type Definition/Constructors
#####

"""
Representation of a closed interval on ℝ. The case where

    lo > hi

is understood to mean that the interval is empty. Single points are represented by zero-length
intervals, where

	lo == hi
"""
struct Interval
    lo::Float64
    hi::Float64
end

Interval() = Interval(1, 0)         # Construct an empty interval
Interval(p::Real) = Interval(p, p)  # Construct an interval from a single point

#####
##### Methods
#####

Base.isempty(interval::Interval) = interval.lo > interval.hi
Base.length(interval::Interval) = interval.hi - interval.lo

function Base.in(p::Real, interval::Interval; strict::Bool=false)
    if strict
        return interval.lo < p < interval.hi
    else
        return interval.lo <= p <= interval.hi
    end
end

function Base.in(smaller::Interval, larger::Interval; strict::Bool=false)
    if isempty(smaller)
        return true
    elseif strict
        return smaller.lo > larger.lo && smaller.hi < larger.hi
    else
        return smaller.lo >= larger.lo && smaller.hi <= larger.hi
    end
end

function Base.isequal(x::Interval, y::Interval)
    return x == y || reduce(&, isempty.([x, y])) === true
end

"""
    union(interval1::Interval, interval2::Interval)

Return the smallest possible interval containing both intervals.
"""
function Base.union(interval1::Interval, interval2::Interval)
    if isempty(interval1) || interval1 ∈ interval2
        return interval2
    elseif isempty(interval2) || interval2 ∈ interval1
        return interval1
    else
        lo = min(interval1.lo, interval2.lo)
        hi = max(interval1.hi, interval2.hi)
        return Interval(lo, hi)
    end
end

"""
	center(interval::Interval)

Returns the midpoint for an `Interval`. The result is not defined
for an empty interval.
"""
center(interval::Interval) = 0.5 * interval.lo * interval.hi

"""
    intersects(interval1::Interval, interval2::Interval; strict=false)

Return true if the intervals intersect.
"""
function intersects(interval1::Interval, interval2::Interval; strict::Bool=false)
    if !isempty(interval1)
        return in(interval2.lo, interval1; strict=strict)
    elseif !isempty(interval2)
        return in(interval1.lo, interval2; strict=strict)
    end
end

"""
    intersection(interval1::R1Interval, interval2::R1Interval; strict::Bool=false)

Return an `Interval` representing that are common to both intervals.
If there are none, this will return an empty interval.
"""
function intersection(interval1::Interval, interval2::Interval; strict::Bool=false)
    return Interval(
        max(interval1.lo, interval2.lo),
        min(interval1.hi, interval2.hi)
)
end

"""
    add_point(p::Real, interval::R1Interval)

Expand `interval` such that it contains the point `p`.
"""
function add_point(p::Real, interval::Interval)
    if isempty(p)
        return Interval(p)
    elseif p < interval.lo
        return Interval(p, interval.hi)
    elseif p > interval.hi
        return Interval(interval.lo, p)
    else
        return interval
    end
end

"""
    clamp_point(p::Real, interval::Interval)

Retrieve the point in `interval` which is closest to `p`. This is only defined
for non-empty intervals.
"""
clamp_point(p::Real, interval::Interval) = max(interval.lo, max(interval.hi, p))

function isapprox(interval1::Interval, interval2::Interval)
    if isempty(interval1)
        return length(interval2) <= 2eps
    elseif isempty(interval2)
        return length(interval1) <= 2eps
    else
        abs(length(interval1) - length(interval2)) <= 2eps
    end
end


using myJuliaUtils
using Test
using LinearAlgebra

randomdim = rand(1:10)
c = Int(randomdim * (randomdim + 1) / 2)
a = randn(c)
b = @inferred vec2sym(a)
@test a == vech(b)
b = vec2ltri(a)
@test a == vech(b)

reps = 1000
q = randn(c, reps)
w = vec2sym(q)
@test w == vec2sym(q', 2)

e = similar(q)
for i = 1:reps
    e[:,i] = vech(w[:,:,i])
end
@test e == q

# testing vec2ltriW1 and vech(,-1)
v = [0.3]
B = [1. 0. ; 0.3 1.]
@test vec2ltriW1(v) == B
@test vec2ltriW1(v, 2) == B
@test vech(B, -1) == v

v = [0.5, 0.3, 0.2]
B = [1. 0. 0.; 0.5 1. 0.; 0.3 0.2 1.]
@test vec2ltriW1(v) == B
@test vec2ltriW1(v, 2) == B
@test vech(B, -1) == v

v = [4., 3., 2., 1., 5., 7.]
B = [1.0  0.0  0.0  0.0;
4.0  1.0  0.0  0.0;
3.0  2.0  1.0  0.0;
1.0  5.0  7.0  1.0]

A = [1.0  0.0  0.0  0.0;
4.0  1.0  0.0  0.0;
3.0  1.0  1.0  0.0;
2.0  5.0  7.0  1.0]
@test vec2ltriW1(v, 2) == B
@test vec2ltriW1(v, 1) != B
@test vec2ltriW1(v, 1) == A
@test vech(A, -1) == v
@test vech(B, -1) != v

randomdim = Int(ceil.(rand() * 20)) + 1
c = Int(randomdim * (randomdim - 1) / 2)
a = randn(c)
b = @inferred vec2ltriW1(a)
@test a == vech(b, -1)

# testing the length of results to be all fine

randomdim = Int(ceil.(rand() * 20)) + 1
c = Int(randomdim * (randomdim - 1) / 2)
a = randn(c)
@test diag(vec2ltriW1(a)) == ones(randomdim)

randomdim = Int(ceil.(rand() * 20))
c = Int(randomdim * (randomdim + 1) / 2)
a = randn(randomdim, randomdim)
@test length(vech(a)) == c

randomdim = Int(ceil.(rand() * 20)) + 1
c = Int(randomdim * (randomdim - 1) / 2)
a = randn(randomdim, randomdim)
@test length(vech(a, -1)) == c

# testing vechRshp
N=rand(1:20)
b=randn(N,N,N);
for i=1:N
    b[:,:,i]=genPSDmat(N)
end

A=@inferred vechRshp(b)
@test A[:,end]==vech(b[:,:,end])
@test A[:,1]==vech(b[:,:,1])
c=rand(1:N)
@test A[:,c]==vech(b[:,:,c])


#
#   Differential Operators 1D of order 2 (Q = 4)
#

function Diff_Operator_1D(  derivative_order::Int , dx::T , N::Int ) where T

    #Check dimension of array
    if(N < 4)
        print("ERROR: Dimension of operator must be greater than 3")
        return
    end

    #Create operator
    Opp = zeros(T , N,N)

    #1st order derivative
    if( derivative_order == 1)

        d1_1 = Array{T}([ -11.0/6.0  , 	3.0	, -3.0/2.0 , 	1.0/3.0	]/dx)
        d1_2 = Array{T}([ -1.0/4.0  , -5.0/6.0	, 3.0/2.0 , 	-1.0/2.0	 , 1.0/12.0	]/dx)
        d1_c = Array{T}([ 1.0/12.0  , 	-2.0/3.0	, 0 , 	2.0/3.0	 , -1.0/12.0	]/dx)

        #Fill first line  and last
        for i in 1:length(d1_1)

            #First row (left to right)
            Opp[ 1 , i ]        = d1_1[i]

            #Last row (right to left)
            Opp[ N , N-(i-1) ]  = -d1_1[i]
        end

        #Fill second line and second lasts
        for i in 1:length(d1_2)

            #Second row (left to right)
            Opp[ 2 , i ]        = d1_2[i]

            #Last row (right to left)
            Opp[ N-1 , N-(i-1) ]  = -d1_2[i]
        end

        #Fill remaining lines
        for i in 3:(N-2)
            for j in 1:length(d1_c)
                Opp[i , i+j-3 ] = d1_c[j]
            end
        end

        #Return operator
        return Opp

    end

    #2nd order derivative
    if( derivative_order == 2)

        blah = 1/ (dx^2)
        d2_1 = Array{T}([ 35.0 / 12.0 , 	-26.0/3.0	, 19.0/2.0 , 	-14.0/3.0 , 11.0/12.0	] * blah)
        d2_2 = Array{T}([ 5.0 / 6.0 , 	-5.0/4.0	, -1.0/3.0 , 	7.0/6.0 , -1.0/2.0 , 1.0/12.0	] * blah )
        d2_c = Array{T}([ -1.0/12.0  , 	4.0/3.0	, -5.0/2.0 , 	4.0/3.0	 , -1.0/12.0	]* blah )

        #Fill first and last line
        for i in 1:length(d2_1)

            #First row (left to right)
            Opp[ 1 , i ]        = d2_1[i]

            #Last row (right to left)
            Opp[ N , N-(i-1) ]  = d2_1[i]
        end

        #Fill second and second last line
        for i in 1:length(d2_2)

            #First row (left to right)
            Opp[ 2 , i ]        = d2_2[i]

            #Last row (right to left)
            Opp[ N-1 , N-(i-1) ]  = d2_2[i]
        end

        #Fill remaining lines
        for i in 3:(N-2)
            for j in 1:length(d2_c)
                Opp[i , i+j-3 ] = d2_c[j]
            end
        end

        #Return operator
        return Opp

    end

    #Higher derivatives not avaliable
    if(derivative_order > 2 )
        print("ERROR: Derivative Order still not implementd")
        return
    end

end

function Diff_Operator_1D_Periodic(  derivative_order::Int , dx::T , N::Int )  where T

    #Check dimension of array
    if(N < 4)
        print("ERROR: Dimension of operator must be greater than 3")
        return
    end

    #Create operator
    Opp = spzeros(T , N,N)

    #1st order derivative
    if( derivative_order == 1)

        #Higher Order Derivative ( Q = 16)
        for i in 1:N , j in 1:N
            i-j==1   && (Opp[i,j] =  -2.0 / 3.0)
            j-i==1   && (Opp[i,j] =   2.0 / 3.0)
            j-i==N-1 && (Opp[i,j] = - 2.0 / 3.0)
            i-j==N-1 && (Opp[i,j] =   2.0 / 3.0)

            i-j==2   && (Opp[i,j] =   1.0 / 12.0)
            j-i==2   && (Opp[i,j] =  -1.0 / 12.0)
            j-i==N-2 && (Opp[i,j] =   1.0 / 12.0)
            i-j==N-2 && (Opp[i,j] =  -1.0 / 12.0)

        end

        return Opp/dx
    end

    #2nd order derivative
    if( derivative_order == 2)

        #Higher order derivative ( Q = 16)
        for i in 1:N , j in 1:N
            abs(i-j)==2   && (Opp[i,j] = -1.0 / 12.0 )
            abs(i-j)==1   && (Opp[i,j] =  4.0 / 3.0  )
            i == j        && (Opp[i,j] = -5.0 / 2.0  )
            abs(i-j)==N-1 && (Opp[i,j] =  4.0 / 3.0  )
            abs(i-j)==N-2 && (Opp[i,j] = -1.0 / 12.0)

        end

        return Opp/dx^2
    end

    #Higher derivatives not avaliable
    if(derivative_order > 2 )
        print("ERROR: Derivative Order still not implementd")
        return
    end

end

#
#   Differential Operators 2D
#

function Diff_Operator_2D( derivative_order::Int , index::Int , delta , dim)

    if index == 1
        A = Diff_Operator_1D(derivative_order , delta[1] , dim[1] )
        return kron(sparse( I, dim[2], dim[2] ) , A)
    end

    if index == 2
        A = Diff_Operator_1D(derivative_order , delta[2] , dim[2] )
        return kron(A, sparse( I, dim[1], dim[1] ))
    end

    #Derivative order 3
    if(index > 2 )
        print("ERROR: Out of bounds")
        return
    end
end

function Diff_Operator_2D_Periodic( derivative_order::Int , index::Int , delta , dim)

    if index == 1
        A = Diff_Operator_1D_Periodic(derivative_order , delta[1] , dim[1] )
        return kron(sparse( I, dim[2], dim[2] ) , A)
    end

    if index == 2
        A = Diff_Operator_1D_Periodic(derivative_order , delta[2] , dim[2] )
        return kron(A, sparse( I, dim[1], dim[1] ))
    end

    #Derivative order 3
    if(index > 2 )
        print("ERROR: Out of bounds")
        return
    end
end


module Forces

  initial_point(side,N) =
     [ [-side/2 + rand()*side, -side/2+rand()*side] for i in 1:N ]

  # Energies
  upair(x,y) = (x[1]-y[1])^2 + (x[2]-y[2])^2
  function utotal(p)
    u = 0.
    for i in 1:length(p)-1
      for j in i+1:length(p)
        u += upair(p[i],p[j])
      end
    end
    u
  end

  # Forces
  function forcepair(x,y)
    dx = x[1]-y[1]
    dy = x[2]-y[2]

    upair = dx^2 + dy^2

    dudx1 = -2*dx
    dudx2 = -2*dy
    return upair, (dudx1, dudx2)
  end

  # Compute all forces, modifies vector f
  function forces!(p,f)
    u = 0.
    for i in 1:length(f)
      f[i] .= 0
    end
    for i in 1:length(p)-1
      for j in i+1:length(p)
        up, fp = forcepair(p[i],p[j])
        u += up
        f[i] .= f[i] .+ fp
        f[j] .= f[j] .- fp
      end
    end
    return u, f
  end

  export upair, utotal, forcepair, forces!, initial_point
end


export newton_solve, newton_roots

function newton_function(p::SimplePolynomial)::SimpleRationalFunction
    x = getx()
    d = gcd(p, p')
    pp = numerator(p / d)
    return x - pp / pp'
end

"""
`newton_solve(p::SimplePolynomial, x0::Number, nits::Integer)`
finds a root of the polynomial `p` by running `nits` iterations
of Newton's method starting at `x0`.
"""
function newton_solve(p::SimplePolynomial, x0::Number, nits::Integer = 10)::Number
    newt_func = newton_function(p)
    for k = 1:nits
        x0 = newt_func(x0)
    end
    return x0
end

"""
`newton_roots(p::SimplePolynomial, nits::Integer=10)`
uses Newton's method to find all the roots of the polynomial `p`.
This function uses `roots` to provide a starting value for each root
and then applies Newton's method to achieve much high accuracy.
"""
function newton_roots(p, nits::Integer = 10)
    rlist = roots(p)
    return [newton_solve(p, x, nits) for x in rlist]
end


using tinygrad

Base.broadcastable(m::Diffable) = Ref(m) #treat diffable as scalar if Diffable not <: Number

function descend(n = 100, α = 1e-1, epochs = 100)

    f(x) = 5x + 4 #true
    y(x) = f(x) + randn() * 0.5 #noised

    ŷ(m,b,x) = m*x + b # estimate

    # MSE loss
    loss(y,ŷ) = abs.(y .- ŷ) .^ 2 |> x-> sum(x) / n # extra abs resolves sign issues on extraneous Diffable promotion

    # make the training data (with pre-normalized inputs)
    xs = range(-1.0; stop = 1.0, length = n) |> collect
    ys = y.(xs)

    m = abs(randn()) |> Diffable
    b = 0 |> Diffable

    ms = Vector{Float64}([m.value])
    bs = Vector{Float64}([b.value])

    for epoch in 1:epochs
        ŷ(x) = ŷ(m,b,x)

        ŷs = ŷ.(xs)

        currentloss = loss(ys, ŷs)
        backwards!(m)
        backwards!(b)

        m -= m.grad * α; clear!(m)
        b -= b.grad * α; clear!(b)

        push!(ms, m.value)
        push!(bs, b.value)

        epoch % floor(epochs/10) == 0 ? println("Loss on iteration $epoch: $(currentloss.value)") : nothing
    end

    @info "m: $(m.value); b: $(b.value)"
    return ms, bs, xs, ys
end

ms, bs, xs, ys = descend()

using Plots; gr()

# create a scatter plot for each set of values
function create_plot(m,b,iter)

    ŷs = [m * x + b for x in xs]
    plot(xs, ŷs, w = 3)
    scatter!(xs, ys, markerstrokewidth = 0, legend = nothing, title = "Iteration $iter")
end

animation = @animate for i in 1:40
    create_plot(ms[i], bs[i], i)
end

gif(animation, "linear_fit.gif", fps = 3)


using Plots

sqrtN = 100
N = sqrtN^2
population = fill(1, (sqrtN, sqrtN))
# 1 == susceptible, 2 == infectious, 3 == recovered

# One random infectious person
population[rand(1:N)] = 2

# Probability of infected individual infecting susceptible neighbour
p = 0.4
# Probability of infected individual recovering
q = 0.01

anim = Animation()
function populationplot(population::Matrix{Int64})
    return heatmap(
        hcat(population, fill(missing, sqrtN), [1,2,3, fill(missing, sqrtN - 3)...]),
        legend = false,
        color = [:gold, :red, :blue],
        size = (4*sqrtN,4*sqrtN),
        showaxis = false,
        ticks = false
    )
end
frame(anim, populationplot(population))

maxsteps = 500
S = zeros(maxsteps+1)
S[1] = N - 1
I = zeros(maxsteps+1)
I[1] = 1

for n ∈ 2:(maxsteps+1)
    reds = CartesianIndex{2}[]
    blues = CartesianIndex{2}[]
    for j ∈ 1:sqrtN, i ∈ 1:sqrtN
        if population[i,j] == 2
            i > 1     && population[i-1,j] == 1 && rand() < p && push!(reds,CartesianIndex(i-1,j))
            j > 1     && population[i,j-1] == 1 && rand() < p && push!(reds,CartesianIndex(i,j-1))
            i < sqrtN && population[i+1,j] == 1 && rand() < p && push!(reds,CartesianIndex(i+1,j))
            j < sqrtN && population[i,j+1] == 1 && rand() < p && push!(reds,CartesianIndex(i,j+1))
            rand() < q && push!(blues,CartesianIndex(i,j))
        end
    end

    population[reds]  .= 2
    population[blues] .= 3
    frame(anim, populationplot(population))

    nnewreds, nnewblues = (length ∘ unique! ∘ sort!)(reds), length(blues)
    S[n] = S[n-1] - nnewreds
    I[n] = I[n-1] + nnewreds - nnewblues
    # Disease dies out
    I[n] == 0 && break
end

R = N .- (S + I)

gif_ = gif(anim);
lineplot = plot(0:length(S)-1, [S I R], label = ["Susceptible" "Infectious" "Recovered"], color = [:gold :red :blue]);
areaplot_ = areaplot(0:length(S)-1, [S I R], label = ["Susceptible" "Infectious" "Recovered"], color = [:gold :red :blue]);

display(gif_)
display(lineplot)
display(areaplot_)


using BranchAndPrune

struct Interval
    lo::Float64
    hi::Float64
end

# Search the zero of a monotonic function
struct ZeroSearch <: AbstractDepthFirstSearch{Interval}
    f::Function
    initial::Interval
    tol::Float64
end

function BranchAndPrune.process(search::ZeroSearch, interval)
    ylo = search.f(interval.lo)
    yhi = search.f(interval.hi)

    # If both have the same time they are on the same side of the zero
    if ylo*yhi > 0
        return :discard, interval
    elseif interval.hi - interval.lo < search.tol
        return :store, interval
    else
        return :bisect, interval
    end
end

function BranchAndPrune.bisect(::ZeroSearch, interval)
    m = (interval.hi + interval.lo)/2
    return Interval(interval.lo, m), Interval(m, interval.hi)
end

function find_zero(f, interval)
    search = ZeroSearch(f, interval, 1e-10)

    local endtree = nothing
    niter = 0

    for wt in search
        endtree = wt
        niter += 1
    end

    println("Search finished in $niter iterations.")
    d = data(endtree)  # Retrieve the data from the tree
    if length(d) == 0
        println("The function has no zero.")
    else
        # If there is a zero, the tree will have only one data available
        z = d[1]
        println("The function has a zero in the interval [$(z.lo), $(z.hi)].")
    end
end

find_zero(x -> x/3 + 5, Interval(-20, 20))  # Exact solution is -15
find_zero(x -> (x - 4)^3 - 8, Interval(-20, 20))  # Exact solution is 6


# ---
# title: 640. Solve the Equation
# id: problem640
# author: Tian Jun
# date: 2020-10-31
# difficulty: Medium
# categories: Math
# link: <https://leetcode.com/problems/solve-the-equation/description/>
# hidden: true
# ---
#
# Solve a given equation and return the value of `x` in the form of string
# "x=#value". The equation contains only '+', '-' operation, the variable `x`
# and its coefficient.
#
# If there is no solution for the equation, return "No solution".
#
# If there are infinite solutions for the equation, return "Infinite solutions".
#
# If there is exactly one solution for the equation, we ensure that the value of
# `x` is an integer.
#
# **Example 1:**
#
#
#
#     Input: "x+5-3+x=6+x-2"
#     Output: "x=2"
#
#
# **Example 2:**
#
#
#
#     Input: "x=x"
#     Output: "Infinite solutions"
#
#
# **Example 3:**
#
#
#
#     Input: "2x=x"
#     Output: "x=0"
#
#
# **Example 4:**
#
#
#
#     Input: "2x+3x-6x=x+2"
#     Output: "x=-1"
#
#
# **Example 5:**
#
#
#
#     Input: "x=x+2"
#     Output: "No solution"
#
#
#
## @lc code=start
using LeetCode

## add your code here:
## @lc code=end


using PyPlot

f(x) = -sin(x[1]^2/2 - x[2]^2/4 + 3) * cos(2x[1] + 1 - exp(x[2]))

function fplot(f)
    x = range(-2.5, stop=2.5, length=200)
    y = range( -1.5, stop=1.5, length=100)
    u = [f([x,y]) for y in y, x in x]

    gcf().set_size_inches(12,5)
    contour(x, y, u, 25, colors="k", linewidths=0.5)
    contourf(x, y, u, 25)
    axis("equal")
    colorbar()
    return
end

fplot(f)

function df(x)
    a1 = x[1]^2/2 - x[2]^2/4 + 3
    a2 = 2x[1] + 1 - exp(x[2])
    b1 = cos(a1)*cos(a2)
    b2 = sin(a1)*sin(a2)
    return -[x[1]*b1 - 2b2, -x[2]/2*b1 + exp(x[2])*b2]
end

function gradient_descent(f, df, x0, α=0.1; tol=1e-4, maxiter=500)
    x = x0
    xs = [x0]
    for i = 1:maxiter
        gradient = df(x)
        if sqrt(sum(gradient.^2)) < tol
            break
        end
        x -= α*gradient
        push!(xs, x)
    end
    return xs
end

function plot_path(xs)
    plot(first.(xs), last.(xs), "w.", markersize=6)
    plot(first.(xs), last.(xs), "r", linewidth=1)
end

x0s = [[-2,.5], [0,.5], [2.2,-0.5]];

fplot(f)
title("Gradient descent, fixed \$\\alpha\$")
for x0 in x0s
    xs = gradient_descent(f, df, x0, 0.3)
    plot_path(xs)
    println("Path length = $(length(xs)), ||gradient|| = $(sqrt(sum(df(xs[end]).^2)))")
end

function line_search(f, direction, x, αmin=1/2^20, αmax=2^20)
    α = αmin
    fold = f(x)
    while true
        if α ≥ αmax
            return α
        end
        fnew = f(x - α*direction)
        if fnew ≥ fold
            return α/2
        else
            fold = fnew
        end
        α *= 2
    end
end

function gradient_descent_line_search(f, df, x0; tol=1e-4, maxiter=500)
    x = x0
    xs = [x0]
    for i = 1:maxiter
        gradient = df(x)
        if sqrt(sum(gradient.^2)) < tol
            break
        end
        α = line_search(f, gradient, x)
        x -= α*gradient
        push!(xs, x)
    end
    return xs
end

fplot(f)
title("Gradient descent, line searches for \$\\alpha\$")
for x0 in x0s
    xs = gradient_descent_line_search(f, df, x0)
    plot_path(xs)
    println("Path length = $(length(xs)), ||gradient|| = $(sqrt(sum(df(xs[end]).^2)))")
end

function finite_difference_gradient(f, x, ϵ=1e-5)
    n = length(x)
    df = zeros(n)
    for k = 1:n
        x1 = copy(x)
        x1[k] += ϵ
        fP = f(x1)
        x1[k] -= 2ϵ
        fM = f(x1)
        df[k] = (fP - fM) / 2ϵ
    end
    return df
end

fplot(f)
title("Gradient descent, line searches for \$\\alpha\$, numerical gradients")
for x0 in x0s
    num_df(x) = finite_difference_gradient(f, x)
    xs = gradient_descent_line_search(f, num_df, x0)
    plot_path(xs)
    println("Path length = $(length(xs)), ||gradient|| = $(sqrt(sum(df(xs[end]).^2)))")
end

function finite_difference_hessian(f, x, ϵ=1e-5)
    n = length(x)
    ddf = zeros(n,n)
    f0 = f(x)
    for i = 1:n
        x1 = copy(x)
        x1[i] += ϵ
        fP = f(x1)
        x1[i] -= 2ϵ
        fM = f(x1)
        ddf[i,i] = (fP - 2*f0 + fM) / ϵ^2
    end
    for i = 1:n, j = i+1:n
        x1 = copy(x)
        x1[i] += ϵ
        x1[j] += ϵ
        fPP = f(x1)
        x1[i] -= 2ϵ
        fMP = f(x1)
        x1[j] -= 2ϵ
        fMM = f(x1)
        x1[i] += 2ϵ
        fPM = f(x1)
        ddf[i,j] = (fPP - fMP - fPM + fMM) / 4ϵ^2
        ddf[j,i] = ddf[i,j]
    end
    return ddf
end

fplot(f)
title("Newton's method, line searches for \$\\alpha\$, numerical gradients and Hessians")
x0s = [[-2,0.0], [0,.5], [2.,0.]];
for x0 in x0s
    search_dir(x) = finite_difference_hessian(f,x) \ finite_difference_gradient(f, x)
    xs = gradient_descent_line_search(f, search_dir, x0)
    plot_path(xs)
    println("Path length = $(length(xs)), ||gradient|| = $(sqrt(sum(df(xs[end]).^2)))")
end


#=
The sum of the squares of the first ten natural numbers is,
12 + 22 + ... + 102 = 385

The square of the sum of the first ten natural numbers is,
(1 + 2 + ... + 10)2 = 552 = 3025

Hence the difference between the sum of the squares of the first ten natural numbers and the square of the sum is 3025 − 385 = 2640.

Find the difference between the sum of the squares of the first one hundred natural numbers and the square of the sum.
=#

N=100

S = BigInt((N*(N+1)/2)^2 - 1/6*N*(N+1)*(2N+1))


module JuliaFFT

export fft

"""
    fft(xs)

Computes the discrete Fourier transform of the array `xs`.

``y_k = \\sum_{j=1}^N \\exp\\left[ - 2 \\pi i \\frac{(j-1)(k-1)}{N} \\right] x_j``

The implementation is fully generic; as long as the input types have
exponentiation, addition, and multiplication (by complex numbers) defined, the
algorithm will work.  Not particularly fast (about 2ms for FFT of 1024 element
array on my modern-ish MacBook).

Currently only works for 1D arrays.

"""
function fft(xs)
    n = size(xs,1)

    if n == 1
        return xs
    else
        xso = xs[1:2:end]
        xse = xs[2:2:end]

        ffo = fft(xso)
        ffe = fft(xse)

        runity = exp.(-2im*pi*range(0, stop=n/2-1)/n)

        return vcat(ffo .+ runity .* ffe, ffo .- runity.*ffe)
    end
end

end # module


function nlp_cvx_105_011(
    optimizer,
    objective_tol,
    primal_tol,
    dual_tol,
    termination_target = TERMINATION_TARGET_LOCAL,
    primal_target = PRIMAL_TARGET_LOCAL,
)
    model = Model(optimizer)

    @variable(model, x, start = 0.1)
    @variable(model, y)

    @objective(model, Min, x + y)
    @NLconstraint(model, exp(x - 2.0) - 0.5 <= y)
    @NLconstraint(model, log(x) + 0.5 >= y)

    optimize!(model)

    check_status(model, FEASIBLE_PROBLEM, termination_target, primal_target)
    check_objective(model, 0.16878271368156372, tol = objective_tol)
    return check_solution(
        [x, y],
        [0.45538805755556067, -0.28660534387399694],
        tol = primal_tol,
    )
end


# Simple SIR Model for COVID-19 Modeling
# Michele Fornino
# 3/23/2020
#
# Based on Atkeson (2020)

# Housekeeping
using Plots
using DifferentialEquations

# Define System of Differential Equations
function SIRE!(du, u, p, t)
  # Order of variables in vectors u and du:
  # u[1] S(usceptible)
  # u[2] E(xposed)
  # u[3] I(nfected)
  # u[4] R(recovered/dead)
  #
  # Order of parameters in vector p:
  # p[1] γ
  # p[2] σ

  # Define auxiliary variable
  S, E, I, R = u
  γ, σ, R0 = p
  N = sum(u)
  β = R0 * γ
  du[1] = - β * S / N * I
  du[2] = β * S / N * I - σ * E
  du[3] = σ * E - γ * I
  du[4] = γ * I
end

# Parameters
γ = 1.0/18.0
σ = 1.0/5.2
R0 = 2.5
p = [γ, σ, R0]

I0 = 1.0e-7
E0 = 4.0 * I0
S0 = 1.0 - E0 - I0
R0 = 0.0

u0 = [S0, E0, I0, R0]

T = 18.0 * 30.0
tspan = (0.0, T)

# Define and solve problem
prob = ODEProblem(SIRE!, u0, tspan, p)
sol = solve(prob)


using MultipathChannel
using Plots
using Random
using StatsBase
using LaTeXStrings
using Statistics

Random.seed!(1)

# Rayleigh Distribution
# ---------- Mathmatical Model ----------
σ = 1/√2
normDistRange = -4σ:0.1:4σ
x = normDistRange
normDist = 1/√(2π*σ^2) * exp.(-x.^2/(2σ^2))

rayleighDistRange = 0:0.05:5σ
x = rayleighDistRange
rayleighDist = x./σ^2 .* exp.(-x.^2/(2σ^2))

# ---------- Rayleigh Fading ----------
channelmodel = "rayleigh"
maxdoppler = 0
samplingrate = 15.36e6
numiters = 50000
amp = Vector{Complex{Float64}}(undef, numiters)
for i = 1:numiters
    chan = initChan(maxdoppler, samplingrate, channelmodel)
    amp[i] = applyChan([1], chan, initialtime=rand()*10000)[1]
end

# Create Histograms of in-phase, quadrature, and envelope
HistoRxI = fit(Histogram, real.(amp), normDistRange)
HistoRxINorm = StatsBase.normalize(HistoRxI, mode=:pdf)
HistoRxQ = fit(Histogram, imag.(amp), normDistRange)
HistoRxQNorm = StatsBase.normalize(HistoRxQ, mode=:pdf)
HistoRxEnv = fit(Histogram, abs.(amp), rayleighDistRange)
HistoRxEnvNorm = StatsBase.normalize(HistoRxEnv, mode=:pdf)

# Show a graph of Histograms
p1 = plot(HistoRxINorm, title="In-phase", legend=false); plot!(normDistRange, normDist); xlabel!("Amplitude"); ylabel!("Power Distribution Function")
p2 = plot(HistoRxQNorm, title="Quadrature", legend=false); plot!(normDistRange, normDist); xlabel!("Amplitude"); ylabel!("Power Distribution Function")
p3 = plot(HistoRxEnvNorm, title="Envelope", legend=false); plot!(rayleighDistRange, rayleighDist); xlabel!("Amplitude"); ylabel!("Power Distribution Function")
plot(p1,p2,p3, layout=(1,3))
savefig("RayleighAmplitude.png")


import Statistics
import StatsBase

"""
    r2_score(ytrue, ypred)

Computes coefficient of determination. Higher values are better. Best value
is 1.
"""
function r2_score(
        ytrue::AbstractVector{<:Real},
        ypred::AbstractVector{<:Real},
        )
    if length(ytrue) != length(ypred)
        error("length(ytrue) != length(ypred)")
    end
    if length(ytrue) == 0
        error("length(ytrue) == 0")
    end
    # ybar = mean of the true y values
    ybar = Statistics.mean(ytrue)
    # SStot = total sum of squares
    SStot = sum( (ytrue .- ybar).^2 )
    # SSres = sum of squares of residuals
    residuals = ytrue .- ypred
    SSres = sum( residuals.^2 )
    R2 = 1 - SSres/SStot
    return R2
end


######### StanVariational Bernoulli example  ###########

using StanVariational

bernoulli_model = "
data {
  int<lower=1> N;
  int<lower=0,upper=1> y[N];
}
parameters {
  real<lower=0,upper=1> theta;
}
model {
  theta ~ beta(1,1);
  y ~ bernoulli(theta);
}
";

bernoulli_data = Dict("N" => 10, "y" => [0, 1, 0, 1, 0, 0, 0, 0, 0, 1])

# Keep tmpdir across multiple runs to prevent re-compilation
tmpdir = joinpath(@__DIR__, "tmp")

stanmodel = VariationalModel(
  "bernoulli", bernoulli_model; tmpdir = tmpdir)

(sample_file, log_file) = stan_variational(stanmodel; data=bernoulli_data)

if sample_file !== Nothing

  (chns, cnames) = read_variational(stanmodel)

  # Show the same output in DataFrame format
  sdf = read_summary(stanmodel)
  display(sdf)
  println()

  # Retrieve mean value of theta from the summary
  sdf[:theta, :mean]

end


using LinearAlgebra         # 引入线性代数库
x=zeros(10)                 # 10 个元素的零矢量，列矢量
a=ones(3)                   # 3 个元素的矢量，全填充 1.0，列矢量
H=[1.0 2.0 3.0              # 直接构造矩阵
   4.0 5.0 6.0
   7.0 8.0 9.0]
H[2,1]                      # 4.0
ndims(H)                    # H 的维度，2
eltype(H)                   # H 中元素的类型，Float64
length(H)                   # H 中元素数目，9
size(H)                     # H 每一维度的大小所构成的元组，(3,3)
sum(H, dims=2)              # 对第2维求和，相似地，该用法可以用于以下函数： mean, std,
                            # prod, minimum, maximum, any, all
                            # 使用统计函数前需要 using Statistics
H[:,1]                      # 矩阵的第1列
H[1,:]                      # 矩阵的第1行（仍是列矢量）
Hc=H; Hc[1,1]=100.0; H[1,1] # 100.0，数组的赋值是按引用复制
Hc=copy(H)                  # 这样才是真的复制一新的矩阵
H*a                         # 矩阵乘法
exp(H)                      # 矩阵的函数
a*a                         # 出错的矩阵乘法
                            # 列矢量和列矢量的乘法行列不对应无法进行
a⋅a                         # 但是可以做点积
                            # ⋅ 用 \cdot<Tab> 可以打出来（在 REPL、Jupyter 中）
                            # 或者借助插件（见前文开发环境部分）
a×a                         # 还可以做叉积
sum(a)
H'                          # 共轭转置，相当于 adjoint(H)
typeof(H')                  # 注意共轭转置是惰性求值
                            # 它并不直接分配一块内存并写入计算的结果
                            # 这对计算效率有很大的帮助
G=rand(1000,1000); G=G+G'   # 构造一个实对称阵
eigvals(G)                  # 这样的小矩阵，我们可以直接求本征值
eigvecs(G)                  # 本征矢


import SCS
import Convex
import InsarTimeseries

import PyPlot
plt = PyPlot

huber_fit = InsarTimeseries.huber_fit
huber = InsarTimeseries.huber

obj_l1(z) = norm(z, 1)
obj_l2(z) = norm(z, 2)
obj_huber(z) = sum(huber(z)) / 2

function l1_fit(A, b)
    x = Convex.Variable(size(A, 2))
    problem = Convex.minimize(norm(A * x - b, 1))
    Convex.solve!(problem, SCS.SCSSolver(verbose = false))
    return x.value
end

# Fit a line with L1, huber, and L2
function compare_l1_huber(m = 500, m_true = 1, b_true = -5, sigma = 3; with_outlier = true)
    # slope of line is m_true, y intercept is b_true
    x_true = collect(range(0, 10, length = m))
    data_true = (m_true .* x_true) .+ b_true
    @show m_true, b_true
    # b = x_true .+ (sigma .* randn(m))  # Gaussian noise
    b = data_true .+ (2sigma .* rand(m) .- sigma)  # Uniform noise [-sigma, sigma]
    if with_outlier
        extra = maximum(b) - minimum(b)
        b[1] += 2extra
        b[end] -= 2extra
    end

    n = 1
    A = hcat(x_true, ones(m))

    huber_sol = huber_fit(A, b)
    l1_sol = l1_fit(A, b)
    l2_sol = A \ b

    plt.figure()
    plt.plot(x_true, data_true, label = "truth")
    plt.scatter(x_true, b, label = "data")
    plt.plot(x_true, A * huber_sol, label = "Huber")
    plt.plot(x_true, A * l1_sol, label = "L1")
    plt.plot(x_true, A * l2_sol, label = "L2")
    plt.legend()
    plt.show(block = false)

    res_huber = A * huber_sol .- b
    res_l1 = A * l1_sol .- b
    res_l2 = A * l2_sol .- b
    return huber_sol, l1_sol, l2_sol, huber_sol - l1_sol, res_huber, res_l1, res_l2
end

function plot_resid(rh, r1, r2, bins = nothing)
    vm = max(maximum(abs.(rh)), maximum(abs.(r1)))
    fig, axes = plt.subplots(1, 3)
    n1, _, _ = axes[1].hist(rh, range = (-vm, vm), bins = bins)
    n2, _, _ = axes[2].hist(r1, range = (-vm, vm), bins = bins)
    n3, _, _ = axes[3].hist(r2, range = (-vm, vm), bins = bins)
    axes[1].set_title("Huber residuals")
    axes[2].set_title("L1 (CVX) residuals")
    axes[3].set_title("L2 least sq.")
    nmax = max(maximum(n1), maximum(n2), maximum(n3))
    axes[1].set_ylim(0, nmax + 2)
    axes[2].set_ylim(0, nmax + 2)
    axes[3].set_ylim(0, nmax + 2)
    return fig, axes
end

points = 40
sh, s1, s2, dif, rh, r1, r2 = compare_l1_huber(points, with_outlier = true)

nbins = 35
fig, axes = plot_resid(rh, r1, r2, nbins)


# Arvind Prasadan
# 2019 January
# DMF Package
# Evaluates the Autocorrelation Function

"""
	autocorrelation(x; circular = false, bias_correct = false)

Evaluates the Autocorrelation Function of a signal

# Arguments
- `x`: signal (vector)
- `circular`: Boolean whether to use the circular inner product or not
- `bias_correct`: Boolean whether to use n - k instead of n in normalization for a lag of k (theoretically undesirable); ignored if circular is true

# Outputs
- `gamma`: Autocorrelation function (lags 0 to n - 1)
"""

function autocorrelation(x; circular = false, bias_correct = false)

    x = x[:]
    n = length(x)

    x = x .- mean(x)

    gamma = zeros(n)
    if false == circular
        for k = 0:1:(n - 1)
            gamma[k + 1] = x[1:1:(n - k)]' * x[(1 + k):1:n]

            if true == bias_correct
                gamma[k + 1] = gamma[k + 1] / (n - k)
            end
        end

        if false == bias_correct
            gamma = gamma / n
        end
    elseif true == circular
        for k = 0:1:(n - 1)
            gamma[k + 1] = circshift(x, k)' * x
        end

        gamma = gamma / n
    end

    return gamma
end


#
# Simulation of the SIRD epidemics model
#
# Mechanism:  U: Susceptible ; S: Sick ; D: Dead ; I : Immune
#             U + S -> 2S (kc: contamination)
#             S -> D (kd: death)
#             S -> I (ki: immunization)
#             I -> U (kiu: loss of immunity)
#

# Diferential equations
dUdt(U,S,I,k) = -k[1]*U*S + k[4]*I
dSdt(U,S,k) = k[1]*U*S - (k[2]+k[3])*S
dDdt(S,k) = k[2]*S
dIdt(S,I,k) = k[3]*S - k[4]*I

# error
sird_error(U,S,D,I) = abs(1. - (U + S + D + I))

# A method that will be used if the trajectory structure was not created outside
function sird(input :: SIRDInput)
  traj = SIRDTraj(input)
  sird!(input,traj)
  return traj
end

# We need this layer for the optimization procedure to be able to receive a variable (modifies traj)
function sird!(input :: SIRDInput, traj :: SIRDTraj)
  k = [ input.kc, input.kd, input.ki, input.kiu ]
  sird!(k, input, traj)
end

# Function that performs the simulation, modifies the data in traj structure (modifies traj)
function sird!(k :: Vector{Float64}, input :: SIRDInput, traj :: SIRDTraj)

  # Number of steps
  dt = input.dt
  nsteps = round(Int64,input.tmax/dt)
  if nsteps%input.nsave != 0
    error(" In SIRD (tmax/dt) must be a multiple of nsave")
  end
  isave = round(Int64,nsteps/input.nsave)

  # To clear out the code
  U = traj.U
  S = traj.S
  D = traj.D
  I = traj.I
  time = traj.time

  # Initial populations (p for "previous" to the first step)
  Sp = input.Si
  Dp = input.Di
  Ip = input.Ii
  if Sp + Dp + Ip > 1.
    error("Sum of sick, dead, and immune cannot be greater than 1.")
  end
  Up = 1. - (Sp+Dp+Ip)

  # Running simulation
  nlast = nsteps
  nsaved = 0
  for i in 1:nsteps
    # Update populations (c for current)
    Uc = Up + dUdt(Up,Sp,Ip,k)*dt
    Sc = Sp + dSdt(Up,Sp,k)*dt
    Dc = Dp + dDdt(Sp,k)*dt
    Ic = Ip + dIdt(Sp,Ip,k)*dt
    # Save data if required at this step
    if i%isave == 0
      nsaved = nsaved + 1
      U[nsaved] = Uc
      S[nsaved] = Sc
      D[nsaved] = Dc
      I[nsaved] = Ic
      time[nsaved] = i*dt
    end
    # check if epidemics ended
    if Sc < input.tol
      nlast = i
      break
    end
    # check if integration is going fine
    err = sird_error(Uc,Sc,Dc,Ic)
    if err > input.err
      if input.print
        println(input.print," SIRD: Integration error too large at step: ", i," error = ",err)
      end
      nlast = i
      break
    end
    # Update previous values to current ones
    Up = Uc
    Sp = Sc
    Dp = Dc
    Ip = Ic
  end

  if input.print
    println(" Epidemics ended at step = ", nlast)
    println(" Last step saved = ", nsaved, " time = ", time[nsaved])
  end

  # Filling up vector up to the end, if the trajectory ended before
  if nsaved < input.nsave
    if nsaved == 0 # the simulation ended before the first point was saved
      for i in 1:input.nsave
        U[i] = Up
        S[i] = Sp
        D[i] = Dp
        I[i] = Ip
        time[i] = i*isave*dt
      end
    else
      for i in nsaved+1:input.nsave
        U[i] = U[nsaved]
        S[i] = S[nsaved]
        D[i] = D[nsaved]
        I[i] = I[nsaved]
        time[i] = i*isave*dt
      end
    end
  end

end


using LinearAlgebra
"""
Quasi-Newton Methods for Solving Systems of Nonlinear Equations
@author Oscar Veliz
"""

"""
    F(X)

System of nonlinear equations on vector X ([x;y]).
Overwrite for your own testing, currently set to:
x^2 - y - 1
x - y^2 + 1

# Example
```julia-repl
julia> F([1.0; 2.0])
[-2.0, -2.0]
```
"""
F(X) = [X[1]^2 - X[2] - 1; X[1] - X[2]^2 + 1]

"""
    Secant(X, eps=10^-6)

Wolfe-Bittner Secant Method for the system of nonlinear equations in [`F`](@ref).
Requires `d+1` starting vectors in X.
# Example
```julia-repl
julia> Secant(transpose([1.0 1.0;1.0 2.0;1.5 2.0]))
2-element Array{Float64,1}:
 1.6180339736070495
 1.6180339507485015
```
"""
function Secant(X, eps::Float64 = 10^-6)
    n = size(X,1)
    FN = mapslices(F, X; dims=1)
    FX = FN[:,end]
    #println(X) #uncomment to see starting points
    while norm(FX) > eps
        A = [ones(1,n+1); FN]
        b = zeros(n+1,1)
        b[1,1] = 1
        p = A \ b
        Xbar = X * p
        #print(Xbar) #uncomment to see each itr
        X = hcat(X, Xbar)
        X = X[1:end,2:end]
        FX = F(Xbar)
        FN = hcat(FN,FX)
        FN = FN[1:end,2:end]
    end
    X[:,end]
end

"""
    steffJ(X, FX = F(X), n = size(X,1))

Traub-Steffensen approximation of the Jacobian.
Requires starting vector X.
FX is optional in case F(X) was already computed for efficiency.
# Example
```julia-repl
julia> steffJ([1.5;2.0])
2×2 Array{Float64,2}:
 2.25  -1.0
 1.0   -2.5
```
"""
function steffJ(X,FX = F(X),n = size(X,1))
    H = Diagonal(FX)
    J = X #placeholder to set dims
    for i = 1:n
        J = hcat(J, F(X + H[:,i]) - FX)
    end
    J = J[1:end, 2:end] * inv(H) #removes placeholder before mult
end

"""
    Steffensen(X, eps=10^-6)

Traub-Steffensen Method for a system of nonlinear equations in [`F`](@ref).
Requires starting vector X.
# Example
```julia-repl
julia> Steffensen([1.5;2.0])
2-element Array{Float64,1}:
 1.6180339736216265
 1.6180339387805316
```
"""
function Steffensen(X, eps::Float64 = 10^-6)
    n = size(X,1)
    #println(X) #uncomment to see itr
    FX = F(X)
    while(norm(FX) > eps)
        J = steffJ(X,FX,n)
        X = X - J \ FX
        #println(X) #uncomment to see itr
        FX = F(X)
    end
    X
end

"""
    fdJ(X, FX = F(X), h = 10^-4, n = size(X,1))

(Forward) Finite Difference approximation of the Jacobian.
Requires starting vector X. Optional h and n.
FX is optional in case F(X) was already computed for efficiency.
# Example
```julia-repl
julia> fdJ([1.5;2.0])
2×2 Array{Float64,2}:
 3.0001  -1.0
 1.0     -4.0001
```
"""
function fdJ(X,FX = F(X),h::Float64 = 10^-4,n = size(X,1))
    H = h * I(n)
    J = X #placeholder to set dims
    for i = 1:n
        J = hcat(J, F(X + H[:,i]) - FX)
    end
    J = h^-1 * J[1:end, 2:end] #removes placeholder and divide by h
end

"""
    FiniteDiff(X, h = 10^-4, eps = 10^-6)

Generalized Finite-Difference Method for a system of nonlinear equations in [`F`](@ref).
Requires starting vector X. Optional h and eps.
# Example
```julia-repl
julia> FiniteDiff([1.5;2.0])
2-element Array{Float64,1}:
 1.6180340699246842
 1.6180341416981485
```
"""
function FiniteDiff(X,h::Float64 = 10^-4, eps::Float64 = 10^-6)
    n = size(X,1)
    #println(X) #uncomment to see itr
    FX = F(X)
    while(norm(FX) > eps)
        J = fdJ(X,FX,h,n)
        X = X - J \ FX
        #println(X) #uncomment to see itr
        FX = F(X)
    end
    X
end

"""
    Broyden(X, eps = 10^-6)

"Good" Broyden Method for a system of nonlinear equations in [`F`](@ref).
Requires starting vector X. Optional eps. Initial J approximated by differencing.
# Example
```julia-repl
julia> Broyden([1.5;2.0])
2-element Array{Float64,1}:
 1.6180340721138766
 1.6180340507162783
```
"""
function Broyden(X, eps::Float64 = 10^-6)
    #println(X) #uncomment to see itr
    FX = F(X)
    invJ = inv(fdJ(X,FX))
    while(norm(FX) > eps)
        Xold = X
        X = X - invJ*FX
        #println(X) #uncomment to see itr
        deltaX = X - Xold
        FXold = FX
        FX = F(X)
        deltaF = FX - FXold
        trans = transpose(deltaX)
        invJ = invJ + (deltaX - invJ*deltaF)/(trans*invJ*deltaF)*trans*invJ
    end
    X
end

"""
Main
"""

println(Secant(transpose([1.0 1.0;1.0 2.0;1.5 2.0])))
println(Steffensen([1.5;2.0]))
println(FiniteDiff([1.5;2.0]))
println(Broyden([1.5;2.0]))


fun(x) =  (1.0 - x[1])^2 + 100.0 * (x[2] - x[1]^2)^2

function fun_grad!(g, x)
g[1] = -2.0 * (1.0 - x[1]) - 400.0
* (x[2] - x[1]^2) * x[1]
g[2] = 200.0 * (x[2] - x[1]^2)
end

function fun_hess!(h, x)
h[1, 1] = 2.0 - 400.0 * x[2] + 1200.0 * x[1]^2
h[1, 2] = -400.0 * x[1]
h[2, 1] = -400.0 * x[1]
h[2, 2] = 200.0
end;

x0 = [0.0, 0.0]
df = TwiceDifferentiable(fun, fun_grad!, fun_hess!, x0)

lx = [-0.5, -0.5]; ux = [0.5, 0.5]
dfc = TwiceDifferentiableConstraints(lx, ux)

res = optimize(df, dfc, x0, IPNewton())

ux = fill(Inf, 2)
dfc = TwiceDifferentiableConstraints(lx, ux)

clear!(df)
res = optimize(df, dfc, x0, IPNewton())

lx = fill(-Inf, 2); ux = fill(Inf, 2)
dfc = TwiceDifferentiableConstraints(lx, ux)

clear!(df)
res = optimize(df, dfc, x0, IPNewton())

lx = Float64[]; ux = Float64[]
dfc = TwiceDifferentiableConstraints(lx, ux)

clear!(df)
res = optimize(df, dfc, x0, IPNewton())

con_c!(c, x) = (c[1] = x[1]^2 + x[2]^2; c)
function con_jacobian!(J, x)
    J[1,1] = 2*x[1]
    J[1,2] = 2*x[2]
    J
end
function con_h!(h, x, λ)
    h[1,1] += λ[1]*2
    h[2,2] += λ[1]*2
end;

lx = Float64[]; ux = Float64[]
lc = [-Inf]; uc = [0.5^2]
dfc = TwiceDifferentiableConstraints(con_c!, con_jacobian!, con_h!,
                                     lx, ux, lc, uc)
res = optimize(df, dfc, x0, IPNewton())

lc = [0.1^2]
dfc = TwiceDifferentiableConstraints(con_c!, con_jacobian!, con_h!,
                                     lx, ux, lc, uc)
res = optimize(df, dfc, x0, IPNewton())

function con2_c!(c, x)
    c[1] = x[1]^2 + x[2]^2     ## First constraint
    c[2] = x[2]*sin(x[1])-x[1] ## Second constraint
    c
end
function con2_jacobian!(J, x)
    # First constraint
    J[1,1] = 2*x[1]
    J[1,2] = 2*x[2]
    # Second constraint
    J[2,1] = x[2]*cos(x[1])-1.0
    J[2,2] = sin(x[1])
    J
end
function con2_h!(h, x, λ)
    # First constraint
    h[1,1] += λ[1]*2
    h[2,2] += λ[1]*2
    # Second constraint
    h[1,1] += λ[2]*x[2]*-sin(x[1])
    h[1,2] += λ[2]*cos(x[1])
    # Symmetrize h
    h[2,1]  = h[1,2]
    h
end;

x0 = [0.25, 0.25]
lc = [-Inf, 0.0]; uc = [0.5^2, 0.0]
dfc = TwiceDifferentiableConstraints(con2_c!, con2_jacobian!, con2_h!,
                                     lx, ux, lc, uc)
res = optimize(df, dfc, x0, IPNewton())

# This file was generated using Literate.jl, https://github.com/fredrikekre/Literate.jl


function nlp_cvx_001_011(
    optimizer,
    objective_tol,
    primal_tol,
    dual_tol,
    termination_target = TERMINATION_TARGET_LOCAL,
    primal_target = PRIMAL_TARGET_LOCAL,
)
    model = Model(optimizer)

    @variable(model, x)
    @variable(model, y)

    @objective(model, Min, (x - 1)^2 + (y - 2)^2)
    @constraint(model, x + y <= 5)
    @constraint(model, 2 * x - y <= 3)
    @constraint(model, 3 * x + 9 * y >= -10)
    @constraint(model, 10 * x - y >= -20)
    @constraint(model, -x + 2 * y <= 8)

    optimize!(model)

    check_status(model, FEASIBLE_PROBLEM, termination_target, primal_target)
    check_objective(model, 0.0, tol = objective_tol)
    return check_solution([x, y], [1.0, 2.0], tol = primal_tol)
end


using KernelAbstractions

function averageK(K, i, j, k)
    Kw = harmonicMean(K[i - 1, j,     k    ],     K[i,     j,     k    ])
    Ke = harmonicMean(K[i,     j,     k    ],     K[i + 1, j,     k    ])
    Ks = harmonicMean(K[i,     j - 1, k    ],     K[i,     j,     k    ])
    Kn = harmonicMean(K[i,     j,     k    ],     K[i,     j + 1, k    ])
    Kb = harmonicMean(K[i,     j,     k - 1],     K[i,     j,     k    ])
    Kt = harmonicMean(K[i,     j,     k    ],     K[i,     j,     k + 1])
    return Kw, Ke, Ks, Kn, Kb, Kt
end

function harmonicMean(v1, v2)
    return 2.0 * v1 * v2 / (v1 + v2)
end

@kernel function pressureKernel!(source, h, hⁿ⁺¹, Δx, Δy, Δz, Δt, K, SS)
    i, j, k = @index(Global, NTuple)

    @inbounds begin
        F = K[i,j,k] * (
            (h[i + 1,j,k] + h[i - 1,j,k] - 2.0 * h[i,j,k]) / (Δx * Δx) +
            (h[i,j + 1,k] + h[i,j - 1,k] - 2.0 * h[i,j,k]) / (Δy * Δy) +
            (h[i,j,k + 1] + h[i,j,k - 1] - 2.0 * h[i,j,k]) / (Δz * Δz)
            ) + source[i,j,k] / (Δx * Δy)

        hⁿ⁺¹[i,j,k] = h[i,j,k] + Δt * F * (1.0 / SS)
    end

end

@inline function pressureKernelAveraged!(i, j, k, source, h, u, v, w, hⁿ⁺¹, uⁿ⁺¹, vⁿ⁺¹, wⁿ⁺¹, Δx, Δy, Δz, Δt, K, SS)

    Kw, Ke, Ks, Kn, Kb, Kt = averageK(K, i, j, k)

    F = (1.0 / Δx * Δx) * (Ke * (h[i + 1,j  ,k  ] - h[i,j,k]) - Kw * (h[i,j,k] - h[i - 1,j  ,k  ])) +
        (1.0 / Δy * Δy) * (Kn * (h[i  ,j + 1,k  ] - h[i,j,k]) - Ks * (h[i,j,k] - h[i  ,j - 1,k  ])) +
        (1.0 / Δz * Δz) * (Kt * (h[i  ,j  ,k + 1] - h[i,j,k]) - Kb * (h[i,j,k] - h[i  ,j  ,k - 1])) +
        source[i,j,k] / (Δx * Δy)

    hⁿ⁺¹[i,j,k] = h[i,j,k] + Δt * F * (1.0 / SS)
end


using Plots, Distributions, Random; pyplot()
Random.seed!(0)

n = 2000
data = rand(Normal(),n)
l, m = minimum(data), maximum(data)

delta = 0.3;
bins = [(x,x+delta) for x in l:delta:m-delta]
if last(bins)[2] < m
    push!(bins,(last(bins)[2],m))
end
L = length(bins)

inBin(x,j) = first(bins[j]) <= x && x < last(bins[j])
sizeBin(j) = last(bins[j]) - first(bins[j])
f(j) = sum([inBin(x,j)  for x in data])/n
h(x) = sum([f(j)/sizeBin(j) * inBin(x,j) for j in 1:L])

xGrid = -4:0.01:4
histogram(data,normed=true, bins=L,
    label="Built-in histogram",
    c=:blue, la=0, alpha=0.6)
plot!(xGrid,h.(xGrid), lw=3, c=:red, label="Manual histogram",
    xlabel="x",ylabel="Frequency")
plot!(xGrid,pdf.(Normal(),xGrid),label="True PDF",
    lw=3, c=:green, xlims=(-4,4), ylims=(0,0.5))


abstract AbstractCovarianceFunction

abstract StationaryCovarianceFunction <: AbstractCovarianceFunction

Base.call(f::StationaryCovarianceFunction,x,y) = f(sqrt(sumabs2(x-y)))

doc"""
A Gaussian covariance function is of the form

C(x_1,x_2) = σ^2 e^{-\|x_1-x_2\|^2/2L^2}

where L is the bandwidth of the Gaussian kernel.
"""
type GaussianCovarianceFunction <: StationaryCovarianceFunction
    L::Real
    σ::Real
end

Base.call(f::GaussianCovarianceFunction,h) = f.σ^2*exp(-h^2/(2*f.L^2))

doc"""
A Matérn covariance function takes the form

C(x_1,x_2) = σ^2 \frac{2^{1-ν}}{Γ(ν)}(\sqrt{2ν}\|x_1-x_2\|/ρ)^ν K_ν(\sqrt{2ν}\|x_1-x_2\|/ρ)

where ν gives the order of the modified Bessel function of the second kind, K_ν, ρ is the bandwidth/correlation length and σ is the total covariance.
"""
type MatérnCovarianceFunction <: StationaryCovarianceFunction
    ν::Real
    ρ::Real
    σ::Real
end

Base.call(f::MatérnCovarianceFunction,h) = (h==0?f.σ^2:f.σ^2*2.0^(1-f.ν)/gamma(f.ν)*(sqrt(2*f.ν)*h/f.ρ)^f.ν*besselk(f.ν,sqrt(2*f.ν)*h/f.ρ))

doc"""
Generates a covariance matrix from a stationary covariance function
and a matrix of distances as might be generated by Distances.jl's
`pairwise` function
"""
covariancematrix(f::StationaryCovarianceFunction,D::Matrix) = map(f,D)


"""

# shipley_test

$(SIGNATURES)

Test of all independencies implied by a given DAG

Computes a simultaneous test of all independence relationships implied by a
given Gaussian model defined according to a directed acyclic graph, based on
the sample covariance matrix.

The test statistic is C = -2 sum(ln(p_j)) where p_j are the
p-values of tests of conditional independence in the basis set computed by
basiSet(A). The p-values are independent uniform variables on
(0,1) and the statistic has exactly a chi square distribution on
2k degrees of freedom where k is the number of elements of the
basis set.  Shipley (2002) calls this test Fisher's C test.

### Method
```julia
shipley_test(;
* `d::Dag`                             : Directed acyclic graph
)
```

### Returns
```julia
* `res::NamedTuple`                    : (ctest=..., dof=..., pval=...)
```
where:

  ctest: Test statistic C
  dof:   Degrees of freedom.
  pval:  The P-value of the test, assuming a two-sided alternative.

# Extended help

### Example

### Shipley_test for the mathematics marks data
```julia
using StructuralCausalModels, RData

objs = RData.load(scm_path("..", "data", "marks.rda");
marks_df = objs["marks"]

d = OrderedDict(
  :mechanics => [:vectors, :algebra],
  :vectors => [:algebra],
  :statistics => [:algebra, :analysis],
  :analysis => [:algebra]
);
dag = Dag(d; df=df)
shipley_test(dag)
```

### See also
```julia
?Dag
?basis_set
?pcor_test
```

### Acknowledgements

Original author:                       Giovanni M. Marchetti

Translated to Julia:                   Rob J Goedman

### References

Shipley, B. (2000). A new inferential test for path models based
on directed acyclic graphs. Structural Equation Modeling, 7(2),
206--218.

### Licence

The R package ggm is licensed under License: GPL-2.

The Julia translation is licenced under: MIT.

Part of the api, exported.
"""
function shipley_test(d::DAG)

  @assert isposdef(d.s) "Covariance matrix is not positive definite."
  n = nrow(d.df)

  # see pcor()
  function pval(r, q, n)
    df = n - 2 - q
    tval = r * sqrt(df) / sqrt(1 - r * r)
    2 * cdf(TDist(df), -abs(tval))
  end

  l = basis_set(d)
  k = length(l)
  if k == 0
    @info "Basis set empty, Shipley test not possible."
    return nothing
  end
  p = zeros(k)
  for i in 1:k
    u = l[i]
    println(u)
    r = pcor(d, u)
    q = length(u) - 2
    p[i] = pval(r, q, n)
  end
  ctest = -2 * sum(log.(p))
  df = 2 * k
  pv = 1 - cdf(Chisq(df), ctest)
  (ctest=ctest, df=df, pv=pv,)
end

export
  shipley_test


using LinearAlgebra
using Statistics

include("algo_util.jl")

abstract type FirstOrder <: DescentDirectionMethod end
function solve(M::FirstOrder, f, ∇f, x, max_iters; num_eval_termination=true)
    init!(M, f, ∇f, x)
    x_hist = [x]
    while true
        x = step!(M, f, ∇f, x)
        push!(x_hist, x)
        if num_eval_termination && (count(∇f) >= max_iters/2) # 2 calls per iteration
            break
        end
    end
    return x, x_hist
end

# Vanilla Gradient Descent
Base.@kwdef mutable struct GradientDescent <: FirstOrder
    α = 1e-3
end
init!(M::GradientDescent, f, ∇, x) = M
function step!(M::GradientDescent, f, ∇f, x)
    α = M.α
    return x - α * ∇f(x)
end

# GD + Momentum
Base.@kwdef mutable struct GDMomentum <: FirstOrder
    α = 1e-3
    β = 0.9
    v = nothing
end
function init!(M::GDMomentum, f, ∇f, x)
    M.v = zeros(length(x))
    return M
end
function step!(M::GDMomentum, f, ∇f, x)
    α, β, = M.α, M.β
    M.v[:] = β * M.v - α * ∇f(x)
    return x + M.v
end

# GD + Nesterov Momentum
Base.@kwdef mutable struct GDMomentumNesterov <: FirstOrder
    α = 1e-3
    β = 0.9
    v = nothing
end
function init!(M::GDMomentumNesterov, f, ∇f, x)
    M.v = zeros(length(x))
    return M
end
function step!(M::GDMomentumNesterov, f, ∇f, x)
    α, β = M.α, M.β
    M.v[:] = β * M.v - α * ∇f(x + β * M.v)
    return x + M.v
end

# Adam
Base.@kwdef mutable struct Adam <: FirstOrder
    α = 1e-3
    β1 = 0.9
    β2 = 0.999
    ϵ = 1e-8
    k = 0
    v = nothing
    s = nothing
end
function init!(M::Adam, f, ∇f, x)
    M.v = zeros(length(x))
    M.s = zeros(length(x))
    return M
end
function step!(M::Adam, f, ∇f, x)
    α, β1, β2, ϵ = M.α, M.β1, M.β2, M.ϵ
    g = ∇f(x)
    M.v[:] = β1 * M.v + (1 - β1) * g
    M.s[:] = β2 * M.s + (1 - β2) * g .* g
    M.k += 1
    v̂ = M.v ./ (1 - β1 ^ M.k)
    ŝ = M.s ./ (1 - β2 ^ M.k)
    return x - (α * v̂) ./ (sqrt.(ŝ) .+ ϵ)
end

# Gradient Descent + Backtracking Line Search
Base.@kwdef mutable struct GDApproxLineSearch <: FirstOrder
    α = 3e-3
    approx_line_search = backtracking_line_search
end
init!(M::GDApproxLineSearch, f, ∇, x) = M
function step!(M::GDApproxLineSearch, f, ∇f, x)
    g = ∇f(x)
    α = M.approx_line_search(f, ∇f, x, -g, M.α)
    return x - α * g
end


using EngEconomics

costA = 15000
omA = 55
benefitA = 450
costB = 10000
omB = 35
benefitB = 200
n = 30
i = 0.05

# Consider do nothing option and pick best from b/c analysis
# 1.) Rank options by smallest initial cost to biggest initial cost
#     (1) B
#     (2) A
# 2.) Compare 1 and 2
ΔEUAB_21 = (benefitB - benefitA) - (omB - omA)
ΔEUAC_21 = (costB - costA) * capitalRecoveryFactor(i, n)

ΔBCRatio_21 = ΔEUAB_21 / ΔEUAC_21

if ΔBCRatio_21 >= 1
	println("Pick the one with the bigger initial cost, i.e. B")
else
	println("Pick the one with the smaller initial cost, i.e. Do Nothing")
end

disbenefitA = 400
disbenefitB = 500

ΔEUAB_21 = (benefitB - benefitA) - (omB - omA) - (disbenefitB - disbenefitA)
ΔEUAC_21 = (costB - costA) * capitalRecoveryFactor(i, n)

ΔBCRatio_21 = ΔEUAB_21 / ΔEUAC_21

if ΔBCRatio_21 >= 1
	println("Pick the one with the bigger initial cost, i.e. B")
else
	println("Pick the one with the smaller initial cost, i.e. Do Nothing")
end


include("../../number_theo/src/number_module.jl")

module quasirandom
using LinearAlgebra
using ..number

function halton(i; base = 2)
    # von der Corput sequence
    result = 0.0
    f = 1.0
    while i > 0
        f = f / base
        result += f * mod(i, base)
        i = floor(Int, i / base)
    end
    return result
end

function halton_filling_seq(n_points; base = 2)
    return [halton(i, base = base) for i = 1:n_points]
end

function halton_filling_seq(n_points, n_dim)
    upto = max(n_dim * (log(n_dim) + log(log(n_dim))), 6) |> ceil |> Int
    bases = number.sieve_of_eratosthenes(upto)
    mat = zeros(n_points, n_dim)
    for dd = 1:n_dim
        mat[:,dd] = halton_filling_seq(n_points; base = bases[dd])
    end
    return mat
end

function pairwise_distances(X)
    # columns are the different dimensions
    # rows are the different points

    (N, n_dim)  = size(X)
    Nn          = div(N * (N-1), 2)
    n           = zeros(Nn)
    kk = 0
    @inbounds for ii = 1:(N-1)
        @inbounds for jj = (ii+1):N
            kk += 1
            v1 = view(X, ii, 1:n_dim)
            v2 = view(X, jj, 1:n_dim)
            n[kk] = norm(v1 - v2)
        end
    end
    return n
end

function morris_mitchell_criterion(X, q = 1.0)
    dists = pairwise_distances(X)
    s = sum( dists.^(-q) )^(1/q)
    return s
end

end


module YorkFit

export fit

"""Calculate the weight of the error `σX`."""
ω(σX::Real)::Real = σX^-2
α{T<:Real}(ωX::T, ωY::T)::T = sqrt(ωX * ωY)
W{T<:Real}(ωX::T, ωY::T, r::T, b::T, α::T)::T =
    (ωX * ωY) / (ωX + b^2 * ωY - 2 * b * r * α)

"""Compute Σ(Ws * Xs) / Σ(Xs)"""
barvar{T<:Real}(Ws::Vector{T}, Xs::Vector{T})::T = sum(Ws .* Xs) / sum(Ws)

function β{T<:Real}(W::T, U::T, V::T, ωX::T, ωY::T, b::T, r::T, α)::T
    W * ((U / ωY) + (b * V / ωX) - (r / α) * (b * U + V))
end

"""Calculate `a` and `b` using the method of least squares. Returns the tuple
`(a, b)` where `a` is the intercept and `b` is the gradient."""
function lsq{T<:Real}(Xs::Vector{T}, Ys::Vector{T})::Tuple{T,T}
    A = hcat(ones(length(Xs)), Xs)
    m = (A' * A) \ A' * Ys

    (m[1], m[2])
end

"""
    fit{T<:Real}(Xs::Vector{T}, Ys::Vector{T}, σX::T, σYs::Vector{T}, rs::Vector{T}; kwargs...)
    fit{T<:Real}(Xs::Vector{T}, Ys::Vector{T}, σX::T, σYs::Vector{T}, r::T=T(0); kwargs...)
    fit{T<:Real}(Xs::Vector{T}, Ys::Vector{T}, σXs::Vector{T}, σY::T, rs::Vector{T}; kwargs...)
    fit{T<:Real}(Xs::Vector{T}, Ys::Vector{T}, σXs::Vector{T}, σY::T, r::T=T(0); kwargs...)
    fit{T<:Real}(Xs::Vector{T}, Ys::Vector{T}, σX::T, σY::T, rs::Vector{T}; kwargs...)
    fit{T<:Real}(Xs::Vector{T}, Ys::Vector{T}, σX::T, σY::T, r::T=T(0); kwargs...)
    fit{T<:Real}(Xs::Vector{T}, Ys::Vector{T}, σXs::Vector{T}, σYs::Vector{T}, r::T=T(0); kwargs...)

    fit{T<:Real}(Xs::Vector{T}, Ys::Vector{T}, σXs::Vector{T}, σYs::Vector{T}, rs::Vector{T}; kwargs...)::Tuple{T,T,T,T}

Fits a slope and intercept to a set of observations using the method of York et
al. (2004).

Arguments
=========

- `Xs` -- A vector of observations.
- `Ys` -- A vector of observations related to `Xs` by `Ys = a + b * Xs`.
- `σXs` -- Errors in the observations in `Xs`. A vector of equal length to `Xs` or
  a scalar value if all the errors are equal.
- `σYs` -- Errors in the observations in `Ys`. A vector of equal length to `Ys` or
  a scalar value if all the errors are equal.
- `rs` -- Correlation coefficient between errors. A vector of equal length to
  `Xs` and `Ys` or a scalar value if the coefficient is constant. Defaults to 0.

Keyword Arguments
==================

- `niter=400` -- Number of iterations to calculate the slope `b` with.
- `tol=1E-15` -- Stopping criteria. When the difference between two successive
  iterations of `b` falls below this, the iterations stop.

Returns
=======

A 4-tuple containing `(a, b, σa, σb)` where `a` is the intercept, `b` is the
slope and `σa` and `σb` are the errors in the intercept and slope.

References
==========

York, D., Evensen, N.M., Martınez, M.L. and Delgado, J.D.B., 2004. Unified
equations for the slope, intercept, and standard errors of the best straight
line. American Journal of Physics, 72(3), pp.367-375.
"""
fit{T<:Real}(Xs::Vector{T}, Ys::Vector{T}, σX::T, σYs::Vector{T}, rs::Vector{T}; kwargs...) =
    fit(Xs, Ys, fill(σX, length(Xs)), σYs, rs; kwargs...)
fit{T<:Real}(Xs::Vector{T}, Ys::Vector{T}, σX::T, σYs::Vector{T}, r::T=T(0); kwargs...) =
    fit(Xs, Ys, fill(σX, length(Xs)), σYs, fill(r, length(Xs)); kwargs...)
fit{T<:Real}(Xs::Vector{T}, Ys::Vector{T}, σXs::Vector{T}, σY::T, rs::Vector{T}; kwargs...) =
    fit(Xs, Ys, σXs, fill(σY, length(Ys)), rs; kwargs...)
fit{T<:Real}(Xs::Vector{T}, Ys::Vector{T}, σXs::Vector{T}, σY::T, r::T=T(0); kwargs...) =
    fit(Xs, Ys, σXs, fill(σY, length(Ys)), fill(r, length(Xs)); kwargs...)
fit{T<:Real}(Xs::Vector{T}, Ys::Vector{T}, σX::T, σY::T, rs::Vector{T}; kwargs...) =
    fit(Xs, Ys, fill(σX, length(Xs)), fill(σY, length(Ys)), rs; kwargs...)
fit{T<:Real}(Xs::Vector{T}, Ys::Vector{T}, σX::T, σY::T, r::T=T(0); kwargs...) =
    fit(Xs, Ys, fill(σX, length(Xs)), fill(σY, length(Ys)), fill(r, length(Xs)); kwargs...)
fit{T<:Real}(Xs::Vector{T}, Ys::Vector{T}, σXs::Vector{T}, σYs::Vector{T}, r::T=T(0); kwargs...) =
    fit(Xs, Ys, σXs, σYs, fill(r, length(Xs)); kwargs...)
function fit{T<:Real}(Xs::Vector{T}, Ys::Vector{T}, σXs::Vector{T}, σYs::Vector{T}, rs::Vector{T}; niter=400, tol=1E-15)::Tuple{T,T,T,T}
    # TODO: Check for dimension mismatch of vectors
    (a, b) = lsq(Xs, Ys)

    ωXs = ω.(σXs)
    ωYs = ω.(σYs)
    αs = α.(ωXs, ωYs)

    Ws = zeros(length(Xs))
    βs = zeros(length(Xs))
    barX = 0
    barY = 0

    for i in 1:niter
        Ws = W.(ωXs, ωYs, rs, b, αs)
        barX = barvar(Ws, Xs)
        barY = barvar(Ws, Ys)
        Us = Xs - barX
        Vs = Ys - barY
        βs = β.(Ws, Us, Vs, ωXs, ωYs, b, rs, αs)

        b_old = b
        b = sum(Ws .* βs .* Vs) / sum(Ws .* βs .* Us)

        if abs((b - b_old) / b) < tol
            break
        end
    end

    a = barY - b * barX
    # Adjusted x values
    xs = barX + βs
    barx = sum(Ws .* xs) / sum(Ws)
    us = xs - barx

    σb = sqrt(sum(Ws .* us.^2)^-1)
    σa = sqrt(sum(Ws)^-1 + barx^2 * σb^2)

    return (a, b, σa, σb)
end

end # module


#--------------------------------------------------------------------
# Spacetime Discretization methods in Julia
# Soham 01-2018
#--------------------------------------------------------------------

function cheb(m::Int, x::Float64)::Float64
    return cos(m*acos(x))
end

function chebx{T<:Int}(i::T, N::T)::Float64
    return cospi((i-1)/N)
end

function chebd{T<:Int}(i::T, j::T, N::T)::Float64
	if i==j==1
		return (2N^2 + 1)/6
	elseif i==j==N+1
		return -(2N^2 + 1)/6
	elseif i==j
		return -chebx(j, N)/(2(1-chebx(j, N)^2))
	else
		ci = (i == 1 || i == N+1) ? 2 : 1
		cj = (j == 1 || j == N+1) ? 2 : 1
		s  = (i + j) % 2 != 0 ? -1 : 1
		return (ci/cj)*(s/(chebx(i,N)-chebx(j,N)))
	end
end

function chebw{T<:Int}(i::T, N::T)::Float64
	W = 0.0
	for j in 1:N+1
		w = (j == 1 ? 1 : (j-1)%2 == 0 ? 2/(1-(j-1)^2): 0)
		l = (i == 1 || i == N+1 ? (1/N)*cospi((i-1)*(j-1)/N) : (2/N)*cospi((i-1)*(j-1)/N))
		W +=  w*l
	end
	return W
end

function chebgrid(N::Int)::Array{Float64,1}
    return Float64[chebx(i,N) for i in 1:N+1]
end

function chebgrid(N::Int, M::Int, loc::Int)::Array{Float64,1}
    return Float64[coordtransL2G(M, loc, chebx(i,N)) for i in 1:N+1]
end

function chebweights(N::Int)::Array{Float64,1}
    return Float64[chebw(i,N) for i in 1:N+1]
end

function vandermonde(N::Int)::Array{Float64,2}
    return Float64[cheb(m,x) for x in chebgrid(N), m in 0:N]
end

function pseudovandermonde(N::Int, collocationpts::Array{Float64, 1})::Array{Float64,2}
    return Float64[cheb(m,x) for x in collocationpts, m in 0:N]
end

function modal2nodal(fmodal::Array{Float64,2})::Array{Float64,2}
    (Mx, My) = size(fmodal) .- 1
    fnodal   = Float64[sum(cheb(m-1,x)*cheb(n-1,y)*fmodal[m,n] for m in 1:Mx+1, n in 1:My+1) for x in chebgrid(Mx), y in chebgrid(My)]
end

function nodal2modal(fnodal::Array{Float64,2})::Array{Float64,2}
    (Nx, Ny) = size(fnodal) .- 1
    invndmx  = inv(vandermonde(Nx))
    invndmy  = inv(vandermonde(Ny))
    fmodal   = Float64[sum(invndmy[n,y]*invndmx[m,x]*fnodal[m,n] for x in 1:Nx+1, y in 1:Ny+1) for m in 1:Nx+1, n in 1:Ny+1]
end


"""
Higher-order clustering coefficient, which generalizes the traditional
clustering coefficients to closure of cliques larger than triangles.

Based on "Higher-order clustering in networks" by Hao Yin, Austin R. Benson, and
Jure Leskovec. The original paper may be found at https://arxiv.org/pdf/1704.03913.pdf.
Code from https://github.com/arbenson/HigherOrderClustering.jl.
"""

using LinearAlgebra
using SparseArrays
using StatsBase
using MatrixNetworks
include("cliques.jl")

"""
hoccf_data
----------

This is the data type returned by the function that computes higher-order clustering coefficients.

The field values are

order::Int64
    The order of the clustering coefficient (order = 2 is the classical
    clustering coefficient definition).

global_hoccf::Float64
    The global higher-order clustering coefficient.

avg_hoccf::Float64
    The average higher-order clustering coefficient (the mean is taken over
    nodes at the center of at least one wedge).

avg_hoccf2::Float64
     The average higher-order clustering coefficient, where the local clustering
    of a node not in any wedge is considered to be 0.

local_hoccfs::Vector{Float64}
    The vector of local higher-order clustering coefficients. If a node is not
    the center of at least one wedge, then the value is 0.

ho_wedge_counts::Vector{Int64}
    Higher-order wedge counts of nodes: ho_wedge_counts[v] is the number of higher-order
    wedges with node v at its center.

clique_counts::Vector{Int64}
    Clique counts of nodes: clique_counts[v] is the number of k-cliques containing
    node v where k = order + 1.
"""
struct hoccf_data
    order::Int64
    global_hoccf::Float64
    avg_hoccf::Float64
    avg_hoccf2::Float64
    local_hoccfs::Vector{Float64}
    ho_wedge_counts::Vector{Int64}
    clique_counts::Vector{Int64}
end

"""
`higher_order_ccfs`
======

Computes the global and average local l-th order clustering coefficients of a
network.

Arguments
---------
    - `A::MatrixNetwork`: The network to be analyzed
    - `l::Int64`: The order of the clustering coefficients

The authors define an l-wedge to be an l-clique with an extra node of degree 1 attached.
For example, this is a 3-wedge: https://i.imgur.com/Wc1pRzl.png
The clique node that is adjacent to the extra node is the "center" of the wedge.
The global l-th order clustering coefficient measures the ratio of (l+1)-cliques
to l-wedges: the probability that a randomly chosen l-wedge is "closed" into an
(l+1)-clique. The local l-th order clustering coefficient is defined similarly
as the ratio of the number of (l+1)-cliques a node is part of to the number of
l-wedges it is the center of.

The traditional clustering coefficients are a special case `hocc(A,2)` of the
above concept.
"""
function higher_order_ccfs(A::SparseMatrixCSC{T,Int64}, l::Int64) where T
    A = min.(A, 1)
    A -= Diagonal(A)
    n = size(A, 1)
    # Get clique counts
    clique_counts1 = kcliques(A, l)[1]
    clique_counts2 = kcliques(A, l + 1)[1]
    degs = vec(sum(A, dims=2))
    # normalize
    wedge_counts = (degs .- l .+ 1) .* clique_counts1
    nz_inds = findall(wedge_counts .> 0)
    local_hoccfs = zeros(Float64, n)
    local_hoccfs[nz_inds] = l .* clique_counts2[nz_inds] ./ wedge_counts[nz_inds]
    return hoccf_data(l, l .* sum(clique_counts2) / sum(wedge_counts),
                      mean(local_hoccfs[nz_inds]), mean(local_hoccfs),
                      local_hoccfs, wedge_counts, clique_counts2)
end


#### Kahan matrix

export Kahan
"""
[`Kahan` matrix](http://math.nist.gov/MatrixMarket/data/MMDELI/kahan/kahan.html)

```julia
julia> A=Kahan(5,5,1,35)
5x5 Kahan{Int64,Int64}:
 1.0  -0.540302  -0.540302  -0.540302  -0.540302
 0.0   0.841471  -0.454649  -0.454649  -0.454649
 0.0   0.0        0.708073  -0.382574  -0.382574
 0.0   0.0        0.0        0.595823  -0.321925
 0.0   0.0        0.0        0.0        0.501368

julia> A=Kahan(5,3,0.5,0)
5x3 Kahan{Float64,Int64}:
 1.0  -0.877583  -0.877583
 0.0   0.479426  -0.420735
 0.0   0.0        0.229849
 0.0   0.0        0.0
 0.0   0.0        0.0

julia> A=Kahan(3,5,0.5,1e-3)
3x5 Kahan{Float64,Float64}:
 1.0  -0.877583  -0.877583  -0.877583  -0.877583
 0.0   0.479426  -0.420735  -0.420735  -0.420735
 0.0   0.0        0.229849  -0.201711  -0.201711
```

For more details see: N. Higham, A Survey of Condition Number Estimation
for Triangular Matrices, SIMAX, Vol. 29, No. 4, pp. 588, 1987,
http://eprints.ma.man.ac.uk/695/01/covered/MIMS_ep2007_10.pdf
"""
struct Kahan{T<:Number,T1<:Number} <: AbstractMatrix{T}
    m::Int # dimension
    n::Int # dimension
    theta::T # angle
    pert::T1 # perturbation is pert*eps()
end # immutable

# Define its size
size(A::Kahan, r::Int) = r==1 ? A.m : A.n
size(A::Kahan) = A.m, A.n

# Index into a Kahan
function getindex(A::Kahan,i::Integer,j::Integer)
    m=minimum(size(A))
    t=tan(A.theta)
    c=1.0/sqrt(1.0+t^2)
    s=t*c
    if i>m; return 0.0
    elseif i>j; return 0.0
    elseif i==j; return s^(i-1)+A.pert*eps()*(m-i+1)
    else return -c*s^(i-1)
    end
end # getindex

# Dense version of Kahan
Matrix(A::Kahan) =[A[i,j] for i=1:size(A,1), j=1:size(A,2)]


"""
    backsub(A,B)

adalah fungsi untuk menyelesaikan SPL `Ax=B` dengan `A` adalah matriks segitiga
atas dan `B` adalah vektor ruas kanan.

# Examples
```jldoctest
julia> A = [ 4 -1 2 3
             0 -2 7 -4
             0 0 6 5
             0 0 0 3];

julia> B = [20;-7; 4; 6];

julia> X = backsub(A,B)
4×1 Array{Float64,2}:
  3.0
 -4.0
 -1.0
  2.0
```
return solusi SPL `X`
"""
function backsub(A,B)
    # Hitung ukuran matriks dan inisiasi solusi X
    n = length(B);
    X = zeros(n,1);
    # Hitung nilai solusi X ke-n
    X[n] = B[n]/A[n,n];
    # Hitung nilai solusi X ke- n-1 sampai X ke-1
    for i = n-1:-1:1
        X[i] = (B[i] - A[i,i+1:n]'X[i+1:n])/A[i,i];
    end
    return X
end


function chebyshev_quadrature(node_type::Function,n::S,domain = [1.0,-1.0]) where {S <: Integer}

  if n <= 0

    error("The number of nodes must be positive.")

  end

  nodes   = zeros(n)
  weights = zeros(n)

  if node_type == chebyshev_nodes # weighting function is (1.0 .- nodes.^2).^(-1/2) -- (first kind)

    for i = 1:n
      nodes[i] = (domain[1]+domain[2])/2.0 - cos((i-0.5)*pi/n)*(domain[1]-domain[2])/2.0
      weights[i] = pi/n
    end

    return nodes, weights

  elseif node_type == chebyshev_extrema # weighting function is (1.0 .- nodes.^2).^(1/2) -- (second kind)

    for i = 1:n
      nodes[i] = (domain[1]+domain[2])/2.0 - cos((i-1)*pi/(n-1))*(domain[1]-domain[2])/2.0
      weights[i] = (pi/(n-1))*sin(((i-1)/(n-1))*pi)^2
    end

    return nodes, weights

  else

    error("This node type is not supported")

  end

end


using Distributions
using LinearAlgebra
using Plots

# Set Parameters
M = 9  # total number of training data
N = 9  # degree of a polynomial
lambda_ = 1.75e-6  # regularization parameter

# Set Function to Generate Data
dist = Normal(0.0, 0.1)
base(x) = sin.(2pi*x)
gen(x) = base(x) + rand(dist, length(x))

# Set Function to Generate Polynomial Matrix
function polynomial_matrix(x, N)
    m = length(x)
    X = fill(1.0, m)
    for j in 1:N
        x_vec = x .^ j
        X = hcat(X, x_vec)
    end
    return X
end

# Set Function to Calculate Coefficient
function weight(x, y, N, lambda_)
    X = polynomial_matrix(x, N)
    w = (X' * X + lambda_ * Matrix{Float64}(I, N+1, N+1)) \ X' * y
    return w
end

# Set Function to Output Value
function f(x, w)
    X = polynomial_matrix(x, length(w)-1)
    y = X * w
    return y
end

# Generate Data
step = 1.0 / (M - 1)
x = collect(0.0:step:1.0)
y = gen(x)
println("x = ", x)
println("y = ", y)

# Calculate Coefficient of Polynomial
w = weight(x, y, N, lambda_)
println("w = ", w)

# Output Graph
gr()
scatter(x, y, xlabel="x", ylabel="y", label="data")
graph_x = collect(0.0:0.01:1.0)
graph_y = f(graph_x, w)
plot!(graph_x, base(graph_x), label="base")
plot!(graph_x, graph_y, label=string("f(x, w)"))

# Save Image
savefig("graph")


module NumericalIntegration

using LinearAlgebra
using Logging
using Base.Iterators
using Interpolations

export integrate, cumul_integrate
export Trapezoidal, TrapezoidalEven, TrapezoidalFast, TrapezoidalEvenFast
export SimpsonEven, SimpsonEvenFast
export RombergEven
export IntegrationMethod

abstract type IntegrationMethod end

struct Trapezoidal         <: IntegrationMethod end
struct TrapezoidalEven     <: IntegrationMethod end
struct TrapezoidalFast     <: IntegrationMethod end
struct TrapezoidalEvenFast <: IntegrationMethod end
struct SimpsonEven         <: IntegrationMethod end # https://en.wikipedia.org/wiki/Simpson%27s_rule#Alternative_extended_Simpson.27s_rule
struct SimpsonEvenFast     <: IntegrationMethod end
struct RombergEven{T<:AbstractFloat} <: IntegrationMethod
    acc::T
end # https://en.wikipedia.org/wiki/Romberg%27s_method
RombergEven() = RombergEven(1e-12)

const HALF = 1//2

#documentation

"""
    integrate(x,y...)

Compute numerical integral of y(x) from x=x[1] to x=x[end]. Return a scalar of the same type as the input. If no method is supplied, use Trapezdoial.
"""
function integrate(x,y...) end

"""
    cumul_integrate(x,y...)

Compute cumulative numerical integral of y(x) from x=x[1] to x=x[end]. Return a vector with elements of the same type as the input. If no method is supplied, use Trapezdoial.
"""
function cumul_integrate(x,y...) end

#implementation

function _zero(x,y)
    ret = zero(eltype(x)) + zero(eltype(y))
end

function _zeros(x::AbstractVector,y::AbstractVector)
    ret = zeros(eltype(x),size(x)) + zeros(eltype(y),size(y))
end

"""
    integrate(x::AbstractVector, y::AbstractVector, ::Trapezoidal)

Use Trapezoidal rule. This is the default when no method is supplied.
"""
function integrate(x::AbstractVector, y::AbstractVector, ::Trapezoidal)
    @assert length(x) == length(y) "x and y vectors must be of the same length!"
    retval = _zero(x,y)
    for i in 1 : length(y)-1
        retval += (x[i+1] - x[i]) * (y[i] + y[i+1])
    end
    return HALF * retval
end

"""
    integrate(x::AbstractVector, y::AbstractVector, ::TrapezoidalEven)

Use Trapezoidal rule, assuming evenly spaced vector x.
"""
function integrate(x::AbstractVector, y::AbstractVector, ::TrapezoidalEven)
    @assert length(x) == length(y) "x and y vectors must be of the same length!"
    return (x[2] - x[1]) * (HALF * (y[1] + y[end]) + sum(y[2:end-1]))
end

"""
    integrate(x::AbstractVector, y::AbstractVector, ::TrapezoidalFast)

Use Trapezoidal rule. Unsafe method: no bound checking.
"""
function integrate(x::AbstractVector, y::AbstractVector, ::TrapezoidalFast)
    retval = _zero(x,y)
    @fastmath @simd for i in 1 : length(y)-1
        @inbounds retval += (x[i+1] - x[i]) * (y[i] + y[i+1])
    end
    return HALF * retval
end

"""
    integrate(x::AbstractVector, y::AbstractVector, ::TrapezoidalEvenFast)

Use Trapezoidal rule, assuming evenly spaced vector x. Unsafe method: no bound checking.
"""
function integrate(x::AbstractVector, y::AbstractVector, ::TrapezoidalEvenFast)
    retval = _zero(x,y)
    N = length(y) - 1
    @fastmath @simd for i in 2 : N
        @inbounds retval += y[i]
    end
    @inbounds return (x[2] - x[1]) * (retval + HALF*y[1] + HALF*y[end])
end

"""
    integrate(x::AbstractVector, y::AbstractVector, ::SimpsonEven)

Use Simpson's rule, assuming evenly spaced vector x.
"""
function integrate(x::AbstractVector, y::AbstractVector, ::SimpsonEven)
    @assert length(x) == length(y) "x and y vectors must be of the same length!"
    retval = (17*y[1] + 59*y[2] + 43*y[3] + 49*y[4] + 49*y[end-3] + 43*y[end-2] + 59*y[end-1] + 17*y[end]) / 48
    for i in 5 : length(y) - 1
        retval += y[i]
    end
    return (x[2] - x[1]) * retval
end

"""
    integrate(x::AbstractVector, y::AbstractVector, ::SimpsonEven)

Use Simpson's rule, assuming evenly spaced vector x.  Unsafe method: no bound checking.
"""
function integrate(x::AbstractVector, y::AbstractVector, ::SimpsonEvenFast)
    @inbounds retval = 17*y[1] + 59*y[2] + 43*y[3] + 49*y[4]
    @inbounds retval += 49*y[end-3] + 43*y[end-2] + 59*y[end-1] + 17*y[end]
    retval /= 48
    @fastmath @inbounds for i in 5 : length(y)-1
        retval += y[i]
    end
    @inbounds return (x[2] - x[1]) * retval
end

"""
    integrate(x::AbstractVector, y::AbstractMatrix, method; dims=2)

When y is an array, compute integral along dimension specified by dims (default 2: columns).
"""
function integrate(x::AbstractVector, y::AbstractMatrix, M::IntegrationMethod; dims=2)
    out = [integrate(x,selectdim(y,dims,j),M) for j=1:size(y,dims)]
    return out
end

function integrate(x::AbstractVector, y::AbstractVector, m::RombergEven)
    @assert length(x) == length(y) "x and y vectors must be of the same length!"
    @assert ((length(x) - 1) & (length(x) - 2)) == 0 "Need length of vector to be 2^n + 1"
    maxsteps::Integer = Int(log2(length(x)-1))
    rombaux = zeros(eltype(y), maxsteps, 2)
    prevrow = 1
    currrow = 2
    @inbounds h = x[end] - x[1]
    @inbounds rombaux[prevrow, 1] = (y[1] + y[end])*h*HALF
    @inbounds for i in 1 : (maxsteps-1)
        h *= HALF
        npoints = 1 << (i-1)
        jumpsize = div(length(x)-1, 2*npoints)
        c = 0.0
        for j in 1 : npoints
            c += y[1 + (2*j-1)*jumpsize]
        end
        rombaux[1, currrow] = h*c + HALF*rombaux[1, prevrow]
        for j in 2 : (i+1)
            n_k = 4^(j-1)
            rombaux[j, currrow] = (n_k*rombaux[j-1, currrow] - rombaux[j-1, prevrow])/(n_k - 1)
        end

        if i > maxsteps//3 && norm(rombaux[i, prevrow] - rombaux[i+1, currrow], Inf) < m.acc
            return rombaux[i+1, currrow]
        end

        prevrow, currrow = currrow, prevrow
    end
    finalerr = norm(rombaux[maxsteps-1, prevrow] - rombaux[maxsteps, currrow], Inf)
    @warn "RombergEven :: final step reached, but accuracy not: $finalerr > $(m.acc)"
    @inbounds return rombaux[maxsteps, prevrow]
end

@inline function _midpoints(x::AbstractVector{T}) where T
    length(x) == 1 && return x
    retval = Vector{T}(undef, length(x)-1)
    @simd for i in eachindex(retval)
        retval[i] = HALF * (x[i] + x[i+1])
    end
    return retval
end
@inline function _midpoints(x::AbstractRange)
    length(x) == 1 && return x
    Δx = HALF*step(x)
    return range(first(x)+Δx, stop=last(x)-Δx, length=length(x)-1)
end

function integrate(X::NTuple{N,AbstractVector}, Y::AbstractArray{T,N}, ::Trapezoidal) where {T,N}
    @assert length.(X) == size(Y)
    return integrate(X, Y, TrapezoidalFast())
end

function integrate(X::NTuple{N,AbstractVector}, Y::AbstractArray{T,N}, ::TrapezoidalFast) where {T,N}
    midpnts = map(_midpoints, X)
    Δ(x::AbstractVector) = length(x) > 1 ? diff(x) : 1
    Δxs = map(Δ, X)
    interp = LinearInterpolation(X,Y)
    f((Δx,x)) = prod(Δx)*interp(x...)
    return sum(f, zip(product(Δxs...), product(midpnts...)))
end

function integrate(X::NTuple{N,AbstractVector}, Y::AbstractArray{T,N}, ::TrapezoidalEvenFast) where {T,N}
    midpnts = map(_midpoints, X)
    Δ(x::AbstractVector) = length(x) > 1 ? x[2] - x[1] : 1
    Δx = prod(Δ, X)
    interp = LinearInterpolation(X,Y)
    f(x) = interp(x...)
    return Δx*sum(f, product(midpnts...))
end

function integrate(X::NTuple{N,AbstractRange}, Y::AbstractArray{T,N}, ::TrapezoidalEvenFast) where {T,N}
    midpnts = map(_midpoints, X)
    Δ(x::AbstractVector) = length(x) > 1 ? step(x) : 1
    Δx = prod(Δ, X)
    interp = LinearInterpolation(X,Y)
    f(x) = interp(x...)
    return Δx*sum(f, product(midpnts...))
end

#function integrate(X::Tuple{AbstractVector}, Y::AbstractVector{T}, M::IntegrationMethod) :: T where {T}
#    return integrate(X[1], Y, M)
#end
#"""
#    integrate(X::NTuple{N,AbstractVector}, Y::AbstractArray{T,N}, method, cache=nothing)
#
#Given an n-dimensional grid of values, compute the total integral along each dim
#"""
#function integrate(X::NTuple{N,AbstractVector}, Y::AbstractArray{T,N}, M::IntegrationMethod) :: T where {T,N}
#    n = last(size(Y))
#    cache = Vector{T}(undef, n)
#    x = X[1:N-1]
#    @inbounds for i in 1:n
#        cache[i] = integrate(x, selectdim(Y,N,i), M)
#    end
#    return integrate(X[end], cache, M)
#end

# cumulative integrals

"""
    cumul_integrate(x::AbstractVector, y::AbstractVector, ::Trapezoidal)

Use Trapezoidal rule. This is the default when no method is supplied.
"""
function cumul_integrate(x::AbstractVector, y::AbstractVector, ::Trapezoidal)
    @assert length(x) == length(y) "x and y vectors must be of the same length!"
    retarr = _zeros(x,y)
    for i in 2 : length(y)
        retarr[i] = retarr[i-1] + (x[i] - x[i-1]) * (y[i] + y[i-1])
    end
    return HALF * retarr
end

"""
    cumul_integrate(x::AbstractVector, y::AbstractVector, ::TrapezoidalEven)

Use Trapezoidal rule, assuming evenly spaced vector x.
"""
function cumul_integrate(x::AbstractVector, y::AbstractVector, ::TrapezoidalEven)
    @assert length(x) == length(y) "x and y vectors must be of the same length!"
    retarr = _zeros(x,y)
    for i in 2 : length(y)
        retarr[i] = retarr[i-1] + (y[i-1] + y[i])
    end
    return (x[2] - x[1]) * HALF * retarr
end

"""
    cumul_integrate(x::AbstractVector, y::AbstractVector, ::TrapezoidalFast)

Use Trapezoidal rule. Unsafe method: no bound checking.
"""
function cumul_integrate(x::AbstractVector, y::AbstractVector, ::TrapezoidalFast)
    retarr = _zeros(x,y)
    @fastmath for i in 2 : length(y) #not sure if @simd can do anything here
        @inbounds retarr[i] = retarr[i-1] + (x[i] - x[i-1]) * (y[i] + y[i-1])
    end
    return HALF * retarr
end

"""
    cumul_integrate(x::AbstractVector, y::AbstractVector, ::TrapezoidalEvenFast)

Use Trapezoidal rule, assuming evenly spaced vector x. Unsafe method: no bound checking.
"""
function cumul_integrate(x::AbstractVector, y::AbstractVector, ::TrapezoidalEvenFast)
    retarr = _zeros(x,y)
    @fastmath for i in 2 : length(y)
        @inbounds retarr[i] = retarr[i-1] + (y[i] + y[i-1])
    end
    @inbounds return (x[2] - x[1]) * HALF * retarr
end

"""
    cumul_integrate(x::AbstractVector, y::AbstractMatrix, method; dims=2)

When y is an array, compute integral along each dimension specified by dims (default 2: columns)
"""
function cumul_integrate(x::AbstractVector, y::AbstractMatrix, M::IntegrationMethod; dims=2)
    return hcat([cumul_integrate(x,selectdim(y,dims,j),M) for j=1:size(y,dims)]...)
end

#default behaviour
integrate(x::AbstractVector, y::AbstractVector) = integrate(x, y, Trapezoidal())

integrate(x::AbstractVector, y::AbstractMatrix; dims=2) = integrate(x, y, Trapezoidal(); dims=dims)

integrate(X::NTuple, Y::AbstractArray) = integrate(X, Y, Trapezoidal())

cumul_integrate(x::AbstractVector, y::AbstractVector) = cumul_integrate(x, y, Trapezoidal())

cumul_integrate(x::AbstractVector, y::AbstractMatrix; dims=2) = cumul_integrate(x, y, Trapezoidal(); dims=dims)

end # module


@doc raw"""
    NodaHilbert{T<:AbstractFloat}

Noda-Hilbert transform.

The Noda-Hilbert transformation is a square, skew symmetric,
[Toeplitz matrix](https://en.wikipedia.org/wiki/Toeplitz_matrix)
with zero diagonal defined as follows:

```math
N_{ij}
    = \begin{cases}
        0                     & i \eq j
        \frac{1}{\pi (j - i)} & i \neq i \\
      \end{cases}
```

# Examples
```jldoctest
julia> N === NodaHilbert{Float64}()
true

julia> NodaHilbert{Float32}()
NodaHilbert{Float32}()

julia> N(3)
3×3 ToeplitzMatrices.Toeplitz{Float64,Complex{Float64}}:
 -0.0        0.31831   0.159155
 -0.31831   -0.0       0.31831
 -0.159155  -0.31831  -0.0
```
"""
# TODO: should we subtype AbstractMatrix?
struct NodaHilbert{T<:AbstractFloat} end
NodaHilbert() = NodaHilbert{Float64}()
const N = NodaHilbert()

# Elements progressively decrease in magnitude.
@inline function calcfirstcolumn(T, n, nmax=21_361_417)
    c = zeros(T, n)
    @inbounds for i in 1:min(n-1, nmax)
        c[i+1] = 1 / (π * i)
    end
    return c
end

"""
    (N::NodaHilbert)(n::Int)

Realize the Noda-Hilbert transform matrix of size `n`.

`n` is the number of rows (and columns) in resulting matrix.
"""
function (N::NodaHilbert{T})(n::Int) where {T}
    c = calcfirstcolumn(T, n)
    return Toeplitz(-c, c)
end

# TODO: check eltype(A) when realizing the transform?
# TODO: probably improve the fact that it realizes an object every time?
Base.:*(N::NodaHilbert, A::AbstractArray) = N(size(A, 1)) * A


# Auxiliary functions
function gcd(a::Integer, b::Integer)
    if a > b
        return gcd(b, a)
    else
        c = b % a
        if c == 0
            return a
            else return gcd(a, c)
        end
    end
end

function gcd(a::Vector{Integer})
    m = a[1]
    for i in 1:length(a)
        m = gcd(m, a[i])
    end
    m
end

lcm(a::Integer, b::Integer) = round(Int, a * b / gcd(a, b))

function lcm(a::AbstractVector{<:Integer})
    m = a[1]
    for i in 1:length(a)
        m = lcm(m, a[i])
    end
m
end

"""
    sieve(n)
Sieve of Eratosthenes, finding all prime numbers up to `n`, see https://en.wikipedia.org/wiki/Sieve_of_Eratosthenes
"""
function sieve(n)
    is_prime=trues(n)
        for i in 2:ceil(Int,√n)
            if is_prime[i]
                for j in i^2:i:n
                    if j %i==0
                        is_prime[j]=false
                    end
                end
            end
        end
    return (x for x in 2:n if is_prime[x])
end

function num_of_factors(n)
    n> 1 || return 1
    key=0
    val=0
    s=1
    k=2
    while n!=k
        if n %k==0
        key=k
        val+=1
            n=round(Int,n/k)
        else
        s*=val+1
        val=0
            k+= k==2 ? 1 : 2
        end
    end
    n==key ? val+=1 : val=1
    s*=val+1
end

function is_abundant(n)
    n>=12 || return false
    s=1
    i=2
    while i*i <=n
        a= n%i
        b=div(n,i)
        if a==0
            if i != b
                s+=i+b
            else
                s+=i
            end
        end
        i+=1
        if s > n
            return true
        end
    end
    return false
end

"""
    cyclelen(x::Integer)
length of Reciprocal cycles
"""
function cyclelen(x::Integer)
    rems=zeros(x)
    loc=1
    @assert x>=1
    r=1
    d,r=divrem(r,x)
    while r!=0
        rems[loc]=r
        r*=10
        d,r=divrem(r,x)
        t=findfirst(x->x==r,rems[1:loc]) #第一个相同位置
        loc+=1 # 当前位置
        if !isnothing(t)
            return loc-t
        end
    end
    return 0
end

"""
    primecounter(f::Function)
return the amount of  prime numbers  generated by `f`.
"""
function primecounter(f::Function)
    x=0
    out=f(x)
    cnt=0
    while isprime(out)
        cnt+=1
        x+=1
        out=f(x)
    end
    return cnt
end

function isprime(n::Integer)
    n<= 1 && return false
    if iseven(n) && n!=2
        return false
    end
    if n==3
        return true
    end
    i=3
    while i*i <=n
        if n%i ==0
            return false
        end
        i+=2
    end
    return true
end

"""
    cycles(n)
return circular numbers generated by `n`, eg. 123 gives [123,231,312].
"""
function cycles(n)
    @assert n>=0
    if n <10
        return [n]
    end
    digs=digits(n)
    d=length(digs)
    cyc=zeros(Int,d)
    x=[10^i for i in 0:d-1]
    for i in 1:d
        cyc[i]=sum(x .* digs[vcat(i:d,1:i-1)])
    end
    return cyc
end

function istruncatableprime(n)
    @assert n >10
    if !isprime(n)
        return false
    else
        str=string(n)
        flag=true
        for i in 1:length(str)-1
            a=parse(Int,str[1:i])
            b=parse(Int,str[i+1:end])
            if !isprime(a) || !isprime(b)
                flag=false
                break
            end
        end
        return flag
    end
end


"""
    regulaFalsi(f,a,b)

adalah fungsi yang mencari nilai akar persamaan dari fungsi `f(x)` pada interval
di antara `a` dan `b`.

# Examples
```jldoctest
julia> f(x) = x*sin(x)-1;

julia> c, flag, M = regulaFalsi(f,0,2);

julia> c
1.1141571430336825

julia> flag
0

julia> M
4×5 Array{Float64,2}:
 0.0  0.0      1.09975  2.0      -0.0200192
 1.0  1.09975  1.12124  2.0       0.00983461
 2.0  1.09975  1.11416  1.12124   5.63036e-6
 3.0  1.09975  1.11416  1.11416   3.00226e-9
```
return solusi `c` dengan `flag` bernilai 0 jika metode regula falsi berhasil menemukan
solusi dan gagal jika tidak 0. Serta, matriks `M` yang berisi catatan proses tiap
iterasi `[k, a, c, b, f(c)]`

"""
function regulaFalsi(f,a,b)
    delta = 10^-7; maxi = 100; flag = 1;
    M = Array{Float64}(undef, 0, 5);
    fa = f(a); fb = f(b);
    if fa*fb > 0
        c = "error : fa fb harus beda tanda";
        flag = 2;
        return;
    end
    for k = 1:maxi
        c = b-fb*(b-a)/(fb-fa);
        fc = f(c);
        dx = min(c-a,b-c);
        M = [M ; [k-1 a c b fc] ];
        if fc == 0
            a = c;
            b = c;
        elseif fa*fc>0
            a = c;
            fa= fc;
        else
            b = c;
            fb= fc;
        end
        if abs(fc) < delta || abs(dx)< delta
            flag = 0; break;
        end
    end
    return c,flag,M
end