# 10/24/2020 © Shuowen Chen
# This script simulates CAPM data using Tauchen & Hussey (1991) method
# as in Wright (2003) and explores sensitivity of Euler equation estimation
# to number of discretization points.

# load packages
using Random, Distributions, LinearAlgebra, Statistics, FastGaussQuadrature, Optim
# set seed
Random.seed!(123)

#======= Defining Functions =======#
# univariate tauchen
function tauchen(m, ρ, N, σ²)
    # INPUTS:
    # 1. m: multiple of unconditional standard deviation
    # 2. ρ: AR(1) coefficient
    # 3. N: number of grid points
    # 4. σ²: variance of the white noise

    # OUTPUTS:
    # 1. Transition probability matrix
    # 2. Sequence of grid points

    # unconditional standard deviation of the AR(1)
    std_un = sqrt(σ²/(1 - ρ^2));
    # max and min of the grid
    ymax = m*std_un;
    ymin = -ymax;
    # length of adjacent grid point
    w = (ymax - ymin)/(N - 1);
    # lay out the grids
    ygrid = collect(ymin:w:ymax);
    # Transition probability matrix (N by N)
    P = zeros(N, N);
    P[:, 1] = cdf.(Normal(0, 1), ((ymin .- ρ*ygrid .+ w/2)./sqrt(σ²)));
    P[:, end] = 1 .- cdf.(Normal(0, 1), ((ymax .- ρ*ygrid .- w/2)./sqrt(σ²)));
    for k = 2:(N - 1)
        P[:, k] = cdf.(Normal(0, 1), (ygrid[k] .- ρ*ygrid .+ w/2)./sqrt(σ²)) -
        cdf.(Normal(0, 1), (ygrid[k] .- ρ*ygrid .- w/2)./sqrt(σ²));
    end
    return(transition = P, grid = ygrid);
end

# Univariate Tauchen and Hussey (1991)
function tauchenhussey(ρ, a, σ², N)
    # PURPOSE:
    # Discretize the following AR(1) process:
    # z_{t+1} = a + ρ*z_{t} + epsilon_{t+1}
    # INPUTS:
    # 1. ρ: AR(1) coefficient
    # 2. a: intercept
    # 3. σ²: variance of epsilon
    # 4. N: number of grid points

    # OUTPUTS:
    # 1. P: transition probability matrix
    # 2. grid: vector of grid points

    # A function from FastGaussQuadrature
    (x, w) = gausshermite(N);
    # Gaussian nodes and weights for normal distribution
    # Acknowledgement: I benefit from reading slides by Karen Kopecky
    # http://www.karenkopecky.net/Teaching/eco613614/Notes_DiscretizingAR1s.pdf

    # unconditional mean of the process
    μ = a / (1 - ρ);
    # Establish the grid using Gaussian-Hermite node
    grid = x .* sqrt(2*σ²) .+ μ;
    # Rescale the Gaussian-Hermite weight
    w = w ./ sqrt(pi);
    # Compute transition probability matrix
    P = zeros(N, N);
    for i = 1:N
        for j = 1:N
            EZprime = a + ρ*grid[i];
            # Gauchen-Hussey weight
            P[i, j] = w[j] * pdf(Normal(EZprime, sqrt(σ²)), grid[j]) /
            pdf(Normal(μ, sqrt(σ²)), grid[j]);
        end
    end
    # Normalization
    for i = 1:N
        P[i, :] = P[i, :] ./ sum(P[i, :])
    end
    return(P, grid);
end

# An auxiliary function to be called in tauchenhusseyvar
function productrule(N, EZ, μ, Σ, grid_j, weight_j, k)
    # Purpose:
    # Compute individual transition prob from state i to j and aggregate
    # by Gauss product rule (tensor product)

    # Inputs:
    # 1. N: number of grid points
    # 2. EZ: expected value in state i (k by 1)
    # 3. μ: unconditional mean of VAR process (k by 1)
    # 4. Σ: var-cov matrix of VAR(1) process
    # 5. grid_j: grid points for the next state
    # 6. weight_j: Gauss-Hermite weight for the next state
    # 7. k: number of variables in the VAR

    # Outputs:
    # 1. prob: aggregate transition probability from state i to j

    den_ind = zeros(1, k);
    for i = 1:k
        den_ind[i] = weight_j/sqrt(pi) * pdf(Normal(EZ[i], diag(sqrt(Σ))[i]),
        grid_j[i]) / pdf(Normal(μ[i], diag(sqrt(Σ))[i]), grid_j[i])
    end
    den_agg = prod(den_ind);
    return(den_agg);
end

# Bivariate Tauchen-Hussey (1991)
function tauchenhusseyvar(k, A, B, Σ, N)
    # PURPOSE: Discretize the following VAR(1) process
    # Y_t = A + B * Y_t-1 + e_t,
    # where e_t ∼ N(0, Σ)
    # Acknowledgement: I benefit from reading the quadrature code by Tauchen,
    # which is available in his website.

    # INPUTS:
    # 1. k: number of variables in the VAR
    # 2. A: constant of VAR(1) (k by 1)
    # 3. B: VAR(1) coefficient matrix (k by k)
    # 4. Σ: iid var-cov matrix
    # 5. N: number of discrete points for each variable k in the VAR

    # OUTPUTS:
    # 1. P: transition matrix (Nᵏ by Nᵏ)
    # 2. Grid: grid points (k by Nᵏ).
    # Note: number of states in the system is equal to number of grids * number of lags,
    # since we consider VAR(1), number of states = number of grid points

    # unconditional mean of the process
    μ = inv(I(k) - B) * A;
    # obtain Gaussian-Hermite nodes and weights
    (x, w) = gausshermite(N);
    # Grid for each variable
    grid_ind = repeat(x, outer = [1, k])' .* sqrt.(2 * diag(Σ)) .+ μ;
    # total grid (k by N^k) and GH weights (N^k) using product rule
    grid = zeros(k, N^k);
    w_tot = zeros(k, N^k);
    for i = 1:k
        grid[i, :] = repeat(grid_ind[i, :], inner = [N^(k-i), 1],
                            outer = [N^(i-1), 1]);
        w_tot[i, :] = repeat(w, inner = [N^(k-i), 1], outer = [N^(i-1), 1]);
    end
    w_tot = prod(w_tot, dims = 1);
    # Compute transition probability matrix
    P = zeros(N^k, N^k);
    for i = 1:N^k
        for j = 1:N^k
            # expected value in state i at time t-1
            EZprime = A + B*grid[:, i]; # k by 1
            # Gauchen-Hussey
            P[i, j] = productrule(N, EZprime, μ, Σ, grid[:, j], w_tot[j], k);
        end
    end
    # Normalization
    for i = 1:N^k
        P[i, :] = P[i, :] ./ sum(P[i, :])
    end
    return(grid, P);
end

# Simulate data using Markov transition probability and grid points
function simulationMC(Pr_mat, grid, T, sim)
    # INPUTS:
    # 1. Pr_mat: transition matrix
    # 2. grid: grid points
    # 3. T: number of observations in each simulation
    # 4. sim: number of simulations
    # 5. k: number of variables

    # OUTPUT:
    # 1. Z: simulated states using MC (k by T for # sim times)
    # Economic meaning: log consumption and dividend growth data
    # 2. state: state index for stock return simulation

    # first obs of the MC (middle state)
    initial_state = convert(Int, floor((length(grid)/size(grid)[1]+1)/2));
    # placeholder for simulated states
    Z = zeros(k, T, sim);
    # initial state
    Z[:, 1, :] .= grid[:, initial_state]; # need to broadcast

    # placeholder for state index
    state = zeros(T, sim);
    # initial state
    state[1, :] .= initial_state;

    # This follows from Adda and Cooper (2003)
    # standard uniformly distributed rv for
    # determining the new state
    u = rand(Uniform(), (T-1, sim));
    # store ∑_{l=1}^{j}π_{i,l} for j = 1,...,N
    Pr_sum = cumsum(Pr_mat, dims = 2);
    old_state = initial_state;
    for i = 1:sim
        for t = 2:T
            new_state = findfirst(u[t-1, i] .<= Pr_sum[old_state, :])
            Z[:, t, i] = grid[:, new_state];
            # store the state index
            state[t, i] = new_state;
            # move to the next obs
            old_state = new_state;
        end
    end
    state = convert.(Int, state);
    return(Z, state);
end

# a function that directly simulates the VAR
function simulationVAR(T, sim, A, B, Σ, k)
    # INPUTS:
    # 1. T: number of observations in each simulation
    # 2. sim: number of simulations
    # 3. A: constant of VAR(1) (k by 1)
    # 4. B: VAR(1) coefficient matrix (k by k)
    # 5. Σ: iid var-cov matrix
    # 6. K: number of variables

    # OUTPUTS:
    # 1. TS: simulated time series (k by 1)

    # normal errors e_t ∼ N(0, Σ) for t = 2,...,T and each simulation
    err = zeros(k, T-1, sim);
    for i = 1:sim
        err[:, :, i] = rand(MvNormal([0; 0], Σ), T-1);
    end
    # placeholder for simulated VAR
    TS = zeros(k, T, sim);
    # initial observation (steady state)
    for i = 1:sim
        TS[:, 1, i] = inv(I(k) - B) * A;
    end
    # Then use the relationship Y_t = A + B*Y_t-1 + e_t
    for i = 1:sim
        for j = 2:T
            # Note the index for err is due to specification of err
            TS[:, j, i] = A + B*TS[:, j-1, i] + err[:, j-1, i];
        end
    end
    return(TS)
end

# Simulate stock return
function simulateReturn(β, γ, P, CG, DG, N, stateindex)
    # INPUTS:
    # 1. β: discount factor
    # 2. γ: CRRA preference
    # 3. P: probability transition matrix
    # 4. CG: consumption growth
    # 5. DG: dividend growth
    # 6. N: number of grid points for each variable

    # OUTPUTS:
    # 1. smreturn: simulated return data

    # Define transition matrix for return
    B = β*P;
    for j = 1:N^2
        B[:, j] = B[:, j] .* CG[j].^(-γ) .* DG[j];
    end
    # State contingent price-dividend ratio (N^2 by 1)
    psm = inv(I(N^2) - B) * B * ones(N^2);
    # Realized stock return from state i to j (N^2 by N^2)
    Rmat = zeros(N^2, N^2);
    for i = 1:N^2
        for j = 1:N^2
            Rmat[i, j] = DG[j]*(1+psm[j])/psm[i];
        end
    end
    # simulate the return data
    T = size(stateindex)[1] - 1;
    smreturn = zeros(T, 1);
    for t = 1:T
        smreturn[t] = Rmat[stateindex[t], stateindex[t+1]];
    end
    return(smreturn);
end

# A function to conduct 2 step GMM
function GMM2step(moments, gradients, k, initial, samplesize)
    # INPUTS:
    # 1. moments: an anonymous function that maps theta to moment conditions
    # 2. gradients: an anoymous function that mapes theta to gradient conditions
    # 3. k: number of moments
    # 4. initial: initial values for estimation (must be floating)
    # 5. samplesize: sample size of the moments
    # OUTPUTS:
    # 1. est: parameter estimate
    # 2. se: standard errors

    # sample average moments
    sample_m = theta -> vec(mean(moments(theta), dims = 1));
    # first step weighting matrix
    W = I(k);
    # objective function in the first step
    obj = theta -> sample_m(theta)'*W*sample_m(theta);
    # minimization using limited BFGS
    firststep = Optim.optimize(obj, initial, LBFGS());
    # second step weighting matrix
    W2 = inv(cov(moments(firststep.minimizer)));
    # objective function in the second step
    obj2 = theta -> sample_m(theta)'*W2*sample_m(theta);
    secondstep = Optim.optimize(obj2, initial, LBFGS());
    # compute the gradient
    # dimension of D: k (# of pars) by p (# of moments)
    D = reshape(mean(gradients(secondstep.minimizer), dims = 1), k, length(initial));
    var = inv(D'*inv(cov(moments(secondstep.minimizer)))*D);
    # compute standard errors
    se = sqrt.(diag(var)./samplesize);
    return(firststep.minimizer, secondstep.minimizer, se);
end

#======= Main Program Starts Here =======#
# Setting parameters
β = 1.139; # utility discount factor
γ = 13.7; # CRRA parameter
k = 2; # number of variables
A = [0.021; 0.04]; # Calibrated values as in Wright (2003)
B = [-0.161 0.017; 0.414 0.117];
Σ = [0.0012 0.00177; 0.00177 0.014];
N = 4; # number of grids for each variable
T = 400; # number of time periods
sim = 400; # number of simulations

# Generate grid points and transition probability
(grid, P) = tauchenhusseyvar(k, A, B, Σ, N);
# Simulate log consumption and dividend growth
(timeseries, stateindex) = simulationMC(P, grid, T, sim);

# Consumption and dividend growth
CG_tot = exp.(timeseries[1, :, :]);
DG_tot = exp.(timeseries[2, :, :]);
# use state index to generate return stock market return
R = zeros(T-1, sim);
for i = 1:sim
    R[:, i] = simulateReturn(β, γ, P, CG_tot[:, i], DG_tot[:, i], N, stateindex[:, i]);
end

# conduct estimation
container = zeros(sim, 2);
for i = 1:sim
    # intercept, lagged return and lagged consumption growth
    ins = [ones(T-2) R[1:end-1, i] CG_tot[1:end-2, i]];
    # sample size of the moments
    t_mom = size(instruments)[1];
    # Construct Euler equation
    euler = theta -> (theta[1] .* R[2:end, i] .* CG_tot[2:end-1, i] .^ (-theta[2]) .-1) .* ins;

    grad = theta -> hcat(R[2:end, i] .* CG_tot[2:end-1, i] .^ (-theta[2]) .* ins,
                             -(theta[1] .* R[2:end, i] .* CG_tot[2:end-1, i] .^ (-theta[2]) .* log.(CG_tot[2:end-1, i])) .* ins);

    gest = GMM2step(euler, grad, 3, [0.0, 13.0], t_mom);
    container[i,:] = gest[2];
end

mean(container, dims =1)


n = 10 # rand(1:10)
A = matrixdepot("randcorr", n)
@test issymmetric(A) # symmetric

# positive semidefinite
for eig in eigvals(A)
    @test eig >= 0
end

# 1s on the diagonal
@test diag(A) ≈ ones(Float64, n)
println("'randcorr' passed test...")


#' ---
#' title: Network Flows
#' ---

#' **Originally Contributed by**: Arpit Bhatia

#' In graph theory, a flow network (also known as a transportation network) is a directed graph where
#' each edge has a capacity and each edge receives a flow. The amount of flow on an edge cannot exceed the capacity of the edge.
#' Often in operations research, a directed graph is called a network, the vertices are called nodes and the edges are called arcs.
#' A flow must satisfy the restriction that the amount of flow into a node equals the amount of flow out of it,
#' unless it is a source, which has only outgoing flow, or sink, which has only incoming flow.
#' A network can be used to model traffic in a computer network, circulation with demands, fluids in pipes,
#' currents in an electrical circuit, or anything similar in which something travels through a network of nodes.

using JuMP
using GLPK

#' ## The Shortest Path Problem
#' Suppose that each arc $(i, j)$ of a graph is assigned a scalar cost $a_{i,j}$, and
#' suppose that we define the cost of a forward path to be the sum of the costs of its arcs.
#' Given a pair of nodes, the shortest path problem is to find a forward path that connects these nodes and has minimum cost.

#' $$
#' \begin{align*}
#' \min && \sum_{\forall e(i,j) \in E} a_{i,j} \times x_{i,j} \\
#' s.t. && b(i) = \sum_j x_{ij} - \sum_k x_{ki} = \begin{cases}
#' 1 &\mbox{if $i$ is the starting node,} \\
#' -1 &\mbox{if $i$ is the ending node,} \\
#' 0 &\mbox{otherwise.} \end{cases} \\
#' && x_{e} \in \{0,1\} && \forall e \in E
#' \end{align*}
#' $$

#'

G = [
0 100 30  0  0;
0   0 20  0  0;
0   0  0 10 60;
0  15  0  0 50;
0   0  0  0  0
]

n = size(G)[1]

shortest_path = Model(with_optimizer(GLPK.Optimizer))

@variable(shortest_path, x[1:n,1:n], Bin)
@constraint(shortest_path, [i = 1:n, j = 1:n; G[i,j] == 0], x[i,j] == 0) # Arcs with zero cost are not a part of the path as they do no exist
@constraint(shortest_path, [i = 1:n; i != 1 && i != 2], sum(x[i,:]) == sum(x[:,i])) # Flow conservation constraint
@constraint(shortest_path, sum(x[1,:]) - sum(x[:,1]) == 1) # Flow coming out of source = 1
@constraint(shortest_path, sum(x[2,:]) - sum(x[:,2]) == -1) # Flowing coming out of destination = -1 i.e. Flow entering destination = 1
@objective(shortest_path, Min, sum(G .* x))

optimize!(shortest_path)
@show objective_value(shortest_path);
@show value.(x);

#' ## The Assignment Problem
#' Suppose that there are $n$ persons and $n$ objects that we have to match on a one-to-one basis.
#' There is a benefit or value $a_{i,j}$ for matching person $i$ with object $j$, and
#' we want to assign persons to objects so as to maximize the total benefit.
#' There is also a restriction that person $i$ can be assigned to object $j$ only if $(i, j)$ belongs to a given set of pairs $A$.
#' Mathematically, we want to find a set of person-object pairs $(1, j_{1}),..., (n, j_{n})$ from $A$ such that
#' the objects $j_{1},...,j_{n}$ are all distinct, and the total benefit $\sum_{i=1}^{y} a_{ij_{i}}$ is maximized.

#' $$
#' \begin{align*}
#' \max && \sum_{(i,j) \in A} a_{i,j} \times y_{i,j} \\
#' s.t. && \sum_{\{j|(i,j) \in A\}} y_{i,j} = 1 && \forall i = \{1,2....n\} \\
#' && \sum_{\{i|(i,j) \in A\}} y_{i,j} = 1 && \forall j = \{1,2....n\} \\
#' && y_{i,j} \in \{0,1\} && \forall (i,j) \in \{1,2...k\}
#' \end{align*}
#' $$

#'

G = [
6 4 5 0;
0 3 6 0;
5 0 4 3;
7 5 5 5;
]

n = size(G)[1]

assignment = Model(with_optimizer(GLPK.Optimizer))
@variable(assignment, y[1:n,1:n], Bin)
@constraint(assignment, [i = 1:n], sum(y[:,i]) == 1) # One person can only be assigned to one object
@constraint(assignment, [j = 1:n], sum(y[j,:]) == 1) # One object can only be assigned to one person
@objective(assignment, Max, sum(G .* y))

optimize!(assignment)
@show objective_value(assignment);
@show value.(y);

#' ## The Max-Flow Problem
#' In the max-flow problem, we have a graph with two special nodes: the $source$, denoted by $s$, and the $sink$, denoted by $t$.
#' The objective is to move as much flow as possible from $s$ into $t$ while observing the capacity constraints.

#' $$
#' \begin{align*}
#' \max && \sum_{v:(s,v) \in E} f(s,v) \\
#' s.t. && \sum_{u:(u,v) \in E} f(u,v)  = \sum_{w:(v,w) \in E} f(v,w) && \forall v \in V - \{s,t\} \\
#' && f(u,v) \leq c(u,v) && \forall (u,v) \in E \\
#' && f(u,v) \geq 0 && \forall (u,v) \in E
#' \end{align*}
#' $$

#'

G = [
0 3 2 2 0 0 0 0
0 0 0 0 5 1 0 0
0 0 0 0 1 3 1 0
0 0 0 0 0 1 0 0
0 0 0 0 0 0 0 4
0 0 0 0 0 0 0 2
0 0 0 0 0 0 0 4
0 0 0 0 0 0 0 0
]

n = size(G)[1]

max_flow = Model(with_optimizer(GLPK.Optimizer))

@variable(max_flow, f[1:n,1:n] >= 0)
@constraint(max_flow, [i = 1:n, j = 1:n], f[i,j] <= G[i,j]) # Capacity constraints
@constraint(max_flow, [i = 1:n; i != 1 && i != 8], sum(f[i,:]) == sum(f[:,i])) # Flow conservation contraints
@objective(max_flow, Max, sum(f[1, :]))

optimize!(max_flow)
@show objective_value(max_flow);
@show value.(f);


"""Returns a matrix of water depth for the whole domain that contains a
Gaussian seamount in the middle. Water depth, heigth and width of the
seamount are adjusted with the constants water_depth, topofeat_height and topofeat_width."""
function seamount()
    xx_T,yy_T = meshgrid(x_T_halo,y_T_halo)
    bumpx = exp.(-((xx_T .- Lx/2).^2)/(2*topofeat_width^2))
    bumpy = exp.(-((yy_T .- Ly/2).^2)/(2*topofeat_width^2))

    H = water_depth .- topofeat_height*bumpx.*bumpy
    return Numtype.(H)
end

"""Returns a matrix of water depth for the whole domain that contains a
meridional Gaussian ridge in the middle. Water depth, heigth and width of the
ridge are adjusted with the constants water_depth, topofeat_height and topofeat_width."""
function ridge()
    xx_T,yy_T = meshgrid(x_T_halo,y_T_halo)
    bumpx = exp.(-((xx_T .- Lx/2).^2)/(2*topofeat_width^2))

    H = water_depth .- topofeat_height*bumpx
    return Numtype.(H)
end

"""Same as ridge() but for 3 ridges at 1/4,1/2,3/4 of the domain."""
function ridges()
    xx_T,yy_T = meshgrid(x_T_halo,y_T_halo)

    # bumps in x direction
    # shift slightly left/right to avoid a symmetric solution
    b0x = exp.(-(xx_T.^2)/(2*topofeat_width^2))
    b1x = exp.(-((xx_T .- 0.99*Lx/4).^2)/(2*topofeat_width^2))
    b2x = exp.(-((xx_T .- 1.01*Lx/2).^2)/(2*topofeat_width^2))
    b3x = exp.(-((xx_T .- 0.99*3*Lx/4).^2)/(2*topofeat_width^2))
    b4x = exp.(-((xx_T .- Lx).^2)/(2*topofeat_width^2))

    th = topofeat_height    # for convenience
    H = water_depth .- th*b0x .- th*b1x .- th*b2x .- th*b3x .- th*b4x
    return Numtype.(H)
end

"""Returns a matrix of constant water depth specified by the constant water_depth."""
function flat_bottom()
    H = fill(water_depth,(nx+2*haloη,ny+2*haloη))
    return Numtype.(H)
end

# rename for convenience
if topography_feature == "ridge"
    topography = ridge
elseif topography_feature == "ridges"
    topography = ridges
elseif topography_feature == "seamount"
    topography = seamount
elseif topography_feature == "flat"
    topography = flat_bottom
else
    throw(error("Topography feature not correctly declared."))
end


"""
    (uvels, vvels, wvels) = streamfunction_xz()

Define a two-dimensional flow field in the x-z directions from a simple streamfunction.
"""
function streamfunction_xz()
    scal = 2e-1
    f(x, y, z) = scal*sin(x*2π/128)*sin(z*π/128) #stream function

    ϕcorners=[f(x,0.,z) for x in 0:128, z in -128:0]
    ϕcenters=[f(x,0.,z) for x in 0.5:128, z in -128:-0.5]

    uu=-diff(ϕcorners,dims=2)[1:end-1,:]
    ww=diff(ϕcorners,dims=1)
    uu=reshape(uu,(128,1,128))
    ww=reshape(ww,(128,1,129))
    uu = reverse(uu, dims=3)
    ww = reverse(ww, dims=3)

    uvels = fill(uu, 2)
    vvels = fill(0*uu, 2)
    wvels = fill(ww, 2)
    uvels = cat(uvels..., dims=4)
    vvels = cat(vvels..., dims=4)
    wvels = cat(wvels..., dims=4)

    return uvels, vvels, wvels, ϕcenters
end

"""
    (uvels, vvels, wvels) = streamfunction_xy()

Define a two-dimensional flow field in the x-y directions from a simple streamfunction
as explained at https://shaddenlab.berkeley.edu/uploads/LCS-tutorial/examples.html#Sec7.1
"""
function streamfunction_xy()
    scal = 3e-1
    f(x, y, z) = scal*(0.3*sin(x*1π/128)*sin(y*2π/128)+0.7*sin(x*2π/128)*sin(y*1π/128)) #stream function

    ϕcorners=[f(x,y,0.) for x in 0:128, y in 0:128]
    ϕcenters=[f(x,y,0.) for x in 0.5:128, y in 0.5:128]

    uu=-diff(ϕcorners,dims=2)[1:end-1,:]
    vv=diff(ϕcorners,dims=1)[:,1:end-1]
    uu=reshape(uu,(128,128,1))
    vv=reshape(vv,(128,128,1))
    ww=zeros(128,128,2)

    uvels = fill(uu, 2)
    vvels = fill(vv, 2)
    wvels = fill(ww, 2)
    uvels = cat(uvels..., dims=4)
    vvels = cat(vvels..., dims=4)
    wvels = cat(wvels..., dims=4)

    return uvels, vvels, wvels, ϕcenters
end


function init_CN(model)
    nx, ny = model.grid.nx, model.grid.ny
    Δx, Δy = model.grid.Δx, model.grid.Δy
    ν, Δt  = model.parameters.ν, model.Δt

    Nx = (nx-2)
    Ny = (ny-2)
    Nxy = Nx*Ny
    βx = 0.5*ν*Δt/(Δx*Δx)
    βy = 0.5*ν*Δt/(Δy*Δy)

    # New a1
    a1 = spzeros(Nxy,Nxy)
    ij_ind = LinearIndices((1:Ny, 1:Nx))
    for ij in ij_ind
        a1[ij,ij] = 1 + 2*βx + 2*βy
    end
    for ij in 1:Nx*(Ny-1)
        a1[view(ij_ind,2:Ny,:)[ij],view(ij_ind,1:Ny-1,:)[ij]] = -βy # Up
        a1[view(ij_ind,1:Ny-1,:)[ij],view(ij_ind,2:Ny,:)[ij]] = -βy # Down
    end
    for ij in 1:(Nx-1)*Ny
        a1[view(ij_ind,:,2:Nx)[ij],view(ij_ind,:,1:Nx-1)[ij]] = -βx # Right
        a1[view(ij_ind,:,1:Nx-1)[ij],view(ij_ind,:,2:Nx)[ij]] = -βx # Left
    end

    # New a2
    a2 = spzeros(Nxy,Nxy)
    ij_ind = LinearIndices((1:Ny, 1:Nx))
    for ij in ij_ind
        a2[ij,ij] = 1 - 2*βx - 2*βy
    end
    for ij in 1:Nx*(Ny-1)
        a2[view(ij_ind,2:Ny,:)[ij],view(ij_ind,1:Ny-1,:)[ij]] = βy # Up
        a2[view(ij_ind,1:Ny-1,:)[ij],view(ij_ind,2:Ny,:)[ij]] = βy # Down
    end
    for ij in 1:(Nx-1)*Ny
        a2[view(ij_ind,:,2:Nx)[ij],view(ij_ind,:,1:Nx-1)[ij]] = βx # Right
        a2[view(ij_ind,:,1:Nx-1)[ij],view(ij_ind,:,2:Nx)[ij]] = βx # Left
    end

    a1_lu = factorize(a1)
    return a1_lu, a2
end
function init_ADI(model)
    nx, ny = model.grid.nx, model.grid.ny
    Δx, Δy = model.grid.Δx, model.grid.Δy
    ν, Δt  = model.parameters.ν, model.Δt

    βx = 0.5*ν*Δt/(Δx*Δx)
    βy = 0.5*ν*Δt/(Δy*Δy)
    d1x = ones(nx-2) .+ 2 * βx
    e1x = -βx * ones(nx-3)
    d1y = ones(ny-2) .+ 2 * βy
    e1y = -βy * ones(ny-3)
    # matrix ( I - 0.5 nu dt delta_x^2 )
    a1x = SymTridiagonal(d1x,e1x)
    a1y = SymTridiagonal(d1y,e1y)

    d2x = ones(nx-2) .- 2 * βx
    e2x = -βx * ones(nx-3)
    d2y = ones(ny-2) .- 2 * βy
    e2y = -βy * ones(ny-3)
    # matrix ( I + 0.5 nu dt delta_x^2)
    a2x = SymTridiagonal(d2x,-e2x)
    a2y = SymTridiagonal(d2y,-e2y)

    #LDLt factorize
    a1x_ldlt = factorize(a1x)
    a1y_ldlt = factorize(a1y)
    return a1x_ldlt, a1y_ldlt, a2x, a2y
end


"""
    poisson_eigenvalues(N, L, dim, ::Periodic)

Return the eigenvalues satisfying the discrete form of Poisson's equation
with periodic boundary conditions along the dimension `dim` with `N` grid
points and domain extent `L`.
"""
function poisson_eigenvalues(N, L, dim, ::Periodic)
    inds = reshape(1:N, reshaped_size(N, dim)...)
    return @. (2sin((inds - 1) * π / N) / (L / N))^2
end

"""
    poisson_eigenvalues(N, L, dim, ::Bounded)

Return the eigenvalues satisfying the discrete form of Poisson's equation
with staggered Neumann boundary conditions along the dimension `dim` with
`N` grid points and domain extent `L`.
"""
function poisson_eigenvalues(N, L, dim, ::Bounded)
    inds = reshape(1:N, reshaped_size(N, dim)...)
    return @. (2sin((inds - 1) * π / 2N) / (L / N))^2
end

"""
    poisson_eigenvalues(N, L, dim, ::Flat)

No need to do any transforms along `Flat` dimensions so just return zeros
since they will be added to the eigenvalues for other dimensions.
"""
poisson_eigenvalues(N, L, dim, ::Flat) = zeros(1, 1, 1)


## In this notebook, we'll look at stochastic gradient descent on a least squares
## system

##
using Plots
pyplot()
theme(:dark)
##
# https://vincentarelbundock.github.io/Rdatasets/csv/carData/Davis.csv
using CSV
df = CSV.read(download("https://vincentarelbundock.github.io/Rdatasets/csv/carData/Davis.csv"))
X = Float64.([df[:weight] df[:height]])
# Filter the data
goodpts = X[:,1] .<= 115
Xf = X[goodpts,:]
##
scatter(Xf[:,1],Xf[:,2],label="",xlabel="weight (kg)", ylabel="height (cm)")
##
x = Xf[:,1]
A = [x ones(length(x))]
b = Xf[:,2]
ab = A\b # solve the least squares problem

##
using Printf
using LinearAlgebra
""" Run stochastic gradient descent to solve a least squares problem. """
function stochastic_gradient_descent_leastsquares(A,b;
            step=0.0001, maxiter=10*size(A,1),tol=1e-6)
  x = zeros(size(A,2)) # start at 0
  for i=1:1000000
    si = rand(1:size(A,1))
    ai = A[si,:]
    bi = b[si]
    g = 2.0.*(ai'*x - bi)*ai
    x -= step*g
  end
  return x
end
stochastic_gradient_descent_leastsquares(A,b)
## Animate the algorithm
quadratic_leastsq(A,b) = x-> (0.5*norm(A*x - b)^2)/size(A,1)
ezsurf(x, y, f) = begin
    X = repeat(x', length(y), 1)
    Y = repeat(y, 1, length(x))
    # Evaluate each f(x, y)
    Z = map((x,y) -> f([x,y]), X, Y)
    plot(x, y, Z, st=:surface)
end
ezsurf(-3:0.1:3,-50:0.5:250,
  quadratic_leastsq(A,b))
plot!(colorbar=false,size=(450,450),dpi=300)
plot!(camera=(170,30))
x = zeros(size(A,2)) # initialize
p = plot3d!(1, label="", marker = 2)
step = 0.0001
anim = Animation()
for i=0:1000000
  global x
  g = (A'*A*x - A'*b)/size(A,1)
  x -= step*g
  if ((i <= 10) || ((i <= 30) & (rem(i,2) == 0)) || ((i <= 100) && rem(i,5) == 0) ||
      ((i <= 1000) && rem(i,10) == 0) || ((i <=10000) && rem(i,100)==0) ||
      ((i <= 100000) && rem(i,1000)==0) || rem(i,10000)==0)
    push!(p[1][2], x[1], x[2], quadratic_leastsq(A,b)(x))
    title!(@sprintf("Iteration %3i  Residual = %.3e", i, norm(g)))
    @printf("Iteration %3i  Residual = %.3e\n", i, norm(g))
    frame(anim)
  end
end
gif(anim, "stochatic-grad-descent-1.gif",fps=30)

##
## Animate the stochastic algorithm
using Random
Random.seed!(0)
quadratic_leastsq(A,b) = x-> (0.5*norm(A*x - b)^2)/size(A,1)
ezsurf(x, y, f) = begin
    X = repeat(x', length(y), 1)
    Y = repeat(y, 1, length(x))
    # Evaluate each f(x, y)
    Z = map((x,y) -> f([x,y]), X, Y)
    plot(x, y, Z, st=:surface)
end
ezsurf(-3:0.1:3,-50:0.5:250,
  quadratic_leastsq(A,b))
plot!(colorbar=false,size=(450,450),dpi=300)
plot!(camera=(170,30))
x = zeros(size(A,2)) # initialize
xgd = zeros(size(A,2)) # initialize
p = plot3d!(1, label="", color=3)
p2 = plot3d!(1, label="", color=2)
step = 0.0001
anim = Animation()
for i=0:1000000
  global x, xgd
  si = rand(1:size(A,1))
  ai = A[si,:]
  bi = b[si]
  g = (ai'*x - bi)*ai
  x -= step*g

  xgd -= step*(A'*A*xgd - A'*b)/size(A,1)
  if ((i <= 10) || ((i <= 30) & (rem(i,2) == 0)) || ((i <= 100) && rem(i,5) == 0) ||
      ((i <= 1000) && rem(i,10) == 0) || ((i <=10000) && rem(i,100)==0) ||
      ((i <= 100000) && rem(i,1000)==0) || rem(i,10000)==0)
    push!(p[1][2], x[1], x[2], quadratic_leastsq(A,b)(x))
    push!(p[1][3], xgd[1], xgd[2], quadratic_leastsq(A,b)(xgd))
    title!(@sprintf("Iteration %3i  Residual = %.3e", i, norm(g)))
    @printf("Iteration %3i  Residual = %.3e\n", i, norm(g))
    frame(anim)
  end
end
gif(anim, "stochatic-grad-descent-2.gif",fps=30)

## Show the quadratic surface
quadratic_leastsq(A,b) = x-> (0.5*norm(A*x - b)^2)
ezsurf(x, y, f) = begin
    X = repeat(x', length(y), 1)
    Y = repeat(y, 1, length(x))
    # Evaluate each f(x, y)
    Z = map((x,y) -> f([x,y]), X, Y)
    plot(x, y, Z, st=:surface)
end
ezsurf(-3:0.1:3,-50:0.5:250,
  quadratic_leastsq(A,b))
##
plot!(colorbar=false,size=(300,300),dpi=300)
anim = @animate for i=1:360
  plot!(camera=(i,30))
end
gif(anim, "leastsq-quadratic.gif")

##
## Show the quadratic surface with alpha-blending
quadratic_leastsq(A,b) = x-> (0.5*norm(A*x - b)^2)
ezsurf(x, y, f) = begin
    X = repeat(x', length(y), 1)
    Y = repeat(y, 1, length(x))
    # Evaluate each f(x, y)
    Z = map((x,y) -> f([x,y]), X, Y)
    plot()
    plot!(x, y, Z, st=:surface)
end
ezsurf(-3:0.1:3,-50:0.5:250,
  quadratic_leastsq(A,b))

##
## Show k random samples of the quadratic surface
plotlyjs()
quadratic_leastsq(A,b) = x-> (0.5*norm(A*x - b)^2)/size(A,1)
ezsurf!(x, y, f) = begin
    X = repeat(x', length(y), 1)
    Y = repeat(y, 1, length(x))
    # Evaluate each f(x, y)
    Z = map((x,y) -> f([x,y]), X, Y)
    plot!(x, y, Z, st=:surface, alpha=0.1)
end
orig = ezsurf(-3:0.25:3,-50:1:250,
  quadratic_leastsq(A,b))
nsamp = 25
k = 25
for i=1:k
  subset = randperm(size(A,1))[1:25] # choose 25 of 200 samples.
  ezsurf!(-3:0.1:3,-50:0.5:250,
    quadratic_leastsq(A[subset,:],b[subset,:]))
end
plot!() # get the plot to show
## Show k random samples of the quadratic surface
using Random
using LinearAlgebra
plotlyjs()
quadratic_leastsq(A,b) = x-> (0.5*norm(A*x - b)^2)/size(A,1)
ezsurf!(x, y, f) = begin
    X = repeat(x', length(y), 1)
    Y = repeat(y, 1, length(x))
    # Evaluate each f(x, y)
    Z = map((x,y) -> f([x,y]), X, Y)
    plot!(x, y, Z, st=:surface, alpha=0.4)
end
plot(framestyle=:none,colorbar=false,size=(300,300),dpi=300)
nsamp = 10
k = 25
for i=1:k
  subset = randperm(size(A,1))[1:nsamp] # choose 25 of 200 samples.
  ezsurf!(-3:0.25:3,-50:2:250,
    quadratic_leastsq(A[subset,:],b[subset,:]))
end
plot!() # get the plot to show
##
#plot!(colorbar=false,size=(300,300),dpi=300)
plot!(colorbar=false,size=(900,900))
anim = @animate for i=2:2:360
  plot!(camera=(i,30))
end
gif(anim, "leastsq-quadratic-samples.gif")

## Show the same thing with a log10
using Random
using LinearAlgebra
plotlyjs()
quadratic_leastsq(A,b) = x-> log10(0.5*norm(A*x - b)^2)/size(A,1)
plot(framestyle=:none,colorbar=false,size=(300,300),dpi=300)
nsamp = 10
k = 25
for i=1:k
  subset = randperm(size(A,1))[1:nsamp] # choose 25 of 200 samples.
  ezsurf!(-3:0.1:3,-50:0.5:250,
    quadratic_leastsq(A[subset,:],b[subset,:]))
end
plot!() # get the plot to show
##
#plot!(colorbar=false,size=(300,300),dpi=300)
plot!(colorbar=false,size=(900,900))
anim = @animate for i=2:2:360
  plot!(camera=(i,30))
end
gif(anim, "leastsq-quadratic-samples-log.gif")
##


input = open("../inputs/day05.txt") do f
    read(f, String)
end

function range(s, low, high)
    lo = low
    hi = high
    for c in s
        if c == 'F' || c == 'L'
            hi = lo + (hi - lo) ÷ 2
        elseif c == 'B' || c == 'R'
            lo = lo + (hi - lo + 1) ÷ 2
        end
    end
    @assert (lo == hi)
    lo
end

function seat_id(s)
    row = range(s[1:7], 0, 127)
    col = range(s[8:10], 0, 7)
    row * 8 + col
end

all_seats = [ seat_id(ln) for ln in split(input, "\n")]
sort!(all_seats)

println("Part 1: ", max(all_seats...))

my_seat_id = first(
    [ all_seats[i] - 1
    for i in 2:length(all_seats)
    if all_seats[i - 1] + 1 != all_seats[i]])

println("Part 2: ", my_seat_id)


"""
    Exponential(lnℓ::AbstractFloat) <: RadialKernel{Euclidean}

The exponential kernel with length scale
``\\exp(\\ln \\ell) = \\ell > 0``.

```math
k_{\\ell}(x, y) = k_{\\ell}(\\lVert x - y\\rVert_2) =
\\exp\\bigg\\lbrace -\\frac{\\lVert x - y\\rVert_2}{\\ell}\\bigg\\rbrace
```
"""
mutable struct Exponential{F<:AbstractFloat} <: RadialKernel{Euclidean}
    dist::Euclidean
    lnℓ::F
    ℓ::F
    function Exponential(dist::Euclidean, lnℓ::AbstractFloat)
        new{typeof(lnℓ)}(dist, lnℓ, exp(lnℓ))
    end
end

Exponential(lnℓ::AbstractFloat) = Exponential(Euclidean(), lnℓ)

@inline (k::Exponential)(τ::AbstractFloat) = exp(-τ / k.ℓ)

numparams(::Exponential) = (1,)
paramtypes(::Exponential{F}) where F = (F,)
params(k::Exponential) = (lnℓ = k.lnℓ,)
function setparams!(k::Exponential{F}, lnℓ::F) where F
    k.lnℓ = lnℓ
    k.ℓ = exp(lnℓ)
end


function hypot(x::DoubleFloat{T}, y::DoubleFloat{T}) where {T<:IEEEFloat}
    ax = abs(x)
    ay = abs(y)
    ay, ax = minmax(ax, ay)

    r = ay
    if !iszero(ax)
        r /= ax
    end
    r *= r

    rr = ax * sqrt(one(DoubleFloat{T}) + r)

    # from Base
    # use type of rr to make sure that return type
    #    is the same for all branches
    if isnan(ay)
        isinf(ax) && return oftype(rr, Inf)
        isinf(ay) && return oftype(rr, Inf)
        return oftype(rr, r)
    end
    return rr
 end

"""
    normalize(x,y)

    x_normalized^2 + y_normalized^2 == one(promote_type(typeof(x),typeof(y)))
"""
function normalize(x::DoubleFloat{T}, y::DoubleFloat{T}) where {T<:IEEEFloat}
    hypotenuse = hypot(x, y)
    if !iszero(hypotenuse) && isfinite(hypotenuse)
        xnorm = x / hypotenuse
        ynorm = y / hypotenuse
    else
        # try for xnorm, ynorm such that xnorm + ynorm == 1
        xnorm = x/(x+y)
        ynorm = y/(x+y)
    end
    return xnorm, ynorm
end


"""
    create_ols_approximation(y::Vector{<:Real}, x::Vector{<:Real}, base_x::Real = 0.0, degree::Integer = 1, intercept::Bool = true)
    create_ols_approximation(y::Vector{<:Real}, x::Union{Vector{DateTime},Vector{Date},Vector{Union{Date,DateTime}}}, base_x::Union{Date,DateTime} = global_base_date, degree::Integer = 1, intercept::Bool = true)

An approximation function calculated via OLS.

### Inputs
* `y` - A `Vector` with the y coordinates
* `x` - A `Vector` with the x coordinates
* `base_x` - A real that offsets the x. So a coordinate with x value of 2.0 will be converted to 1.8 if base_x is 0.2.
* `degree` - What the highest power of x should be. So if this is 3 then the equation will have x, x^2, x^3 as predictors.
* `intercept` - Should there be an x intercept.
### Returns
* A `Sum_Of_Functions` containing the approximation function.
"""
function create_ols_approximation(y::Vector{<:Real}, x::Vector{<:Real}, base_x::Real = 0.0, degree::Integer = 1, intercept::Bool = true)
    obs = length(y)
    if degree < 0
        error("Cannot approximate with OLS with a degree that is negative")
    end
    x = x .- base_x
    if intercept
        X = ones(obs)
        for i in 1:degree
            X = hcat(X, (x .^ i))
        end
    else
        X = x
        for i in 2:degree
            X = hcat(X, (x .^ i))
        end
    end

    lm1 = fit(LinearModel,  hcat(X), y)
    beta = lm1.pp.beta0
    func_array = Array{PE_Function,1}(undef,convert(Int, intercept) + degree)
    if intercept
        func_array[1] = PE_Function(beta[1], 0.0, base_x, 0)
    end
    for d in 1:degree
        func_array[d+Integer(intercept)] = PE_Function(beta[d+Integer(intercept)], 0.0, base_x, d)
    end
    return Sum_Of_Functions(func_array)
end

function create_ols_approximation(y::Vector{<:Real}, x::Union{Vector{DateTime},Vector{Date},Vector{Union{Date,DateTime}}}, base_x::Union{Date,DateTime} = global_base_date, degree::Integer = 1, intercept::Bool = true)
    base = years_from_global_base.(base_x)
    xx   = years_from_global_base.(x)
    return create_ols_approximation(y, xx, base, degree, intercept)
end

function get_cholesky_coefficients(chebyshev::Sum_Of_Functions, y::Vector{<:Real}, unnormalised_nodes::Vector{<:Real})
    chebyshev_on_nodes = evaluate.(Ref(chebyshev), unnormalised_nodes)
    a = sum(y .* chebyshev_on_nodes) / sum(chebyshev_on_nodes .^ 2)
    return a
end

"""
    create_chebyshev_approximation(func::Function, nodes::Integer, degree::Integer, left::Real, right::Real)

An function that will approximate another function via Chebyshev polynomials.

### Inputs
* `func` - A function that you want to approximation
* `nodes` - The number of approximation nodes
* `degree` - The degree of the Chebyshev polynomials.
* `left` - The left limit of the approximation
* `right` - The right limit of the approximation.
### Returns
* A `Sum_Of_Functions` containing the approximation function.
"""
function create_chebyshev_approximation(func::Function, nodes::Integer, degree::Integer, left::Real, right::Real)
    # This is all after Algorithm 6.2 from Judd (1998) Numerical Methods in Economics.
    if nodes <= degree
        error("Need to have more nodes than degree to use a chebyshev approximation")
    end
    k = 1:nodes
    unnormalised_nodes = -cos.( (((2 .* k) .- 1) ./ (2 * nodes)) .* pi    )
    normalised_nodes = ((unnormalised_nodes .+ 1) .* ((right-left)/2)) .+ left
    y = func.(normalised_nodes)
    chebyshevs = get_chevyshevs_up_to(degree, true)
    a = get_cholesky_coefficients.(chebyshevs, Ref(y), Ref(unnormalised_nodes))
    transformed_chebyshevs = convert_to_linearly_rescale_inputs.(chebyshevs, (right-left)/2, +(right+left)/2)
    # Note that these alpha and beta parameters in the convert_to_linearly_rescale_inputs function differ from those in Judd because those did not work in this context.
    all_terms = a .* transformed_chebyshevs
    final_func = Sum_Of_Functions(all_terms)
    return final_func
end


"""
    psMF(lnν)

Press-Schechter halo multiplicity function.

*Reference*: Press & Schechter, 187, 425 (1974)
- See also Bond, Cole, Efstathiou & Kaiser, ApJ, 379, 440 (1991)
- See Sheth & Tormen, MNRAS, 329, 61 (2002) for the notation used here. `psMF(lnν)` corresponds to νf(ν) in this reference.

# Arguments
- `lnν::Real`: natural logarithm of a threshold, ν, i.e., `lnν` = log(ν), defined by ν ≡ [δc/σ(R)]^2. Here, δc = 1.6865 and σ(R) is the r.m.s. mass fluctuation within a top-hat smoothing of scale R.

Press & Schechter's multiplicity function is normalized, i.e,

``∫_-∞^∞ dlnν psMF(lnν) = 1``

This function is based on [Cosmology Routine Library (CRL)](https://wwwmpa.mpa-garching.mpg.de/~komatsu/crl/).
"""
function psMF(lnν::Real)
    ν = exp(lnν)
    mf = exp(-ν / 2) * √(ν / 2π)
end

"""
    stMF(lnν)

Sheth-Tormen halo multiplicity function.

*Reference*: Equation (2) of Sheth & Tormen, MNRAS, 329, 61 (2002)
- See also Sheth, Mo & Tormen, MNRAS, 323, 1 (2001)
- `stMF(lnν)` corresponds to νf(ν) in these references

# Arguments
- `lnν::Real`: natural logarithm of a threshold, ν, i.e., `lnν` = log(ν), defined by ν ≡ [δc/σ(R)]^2. Here, δc = 1.6865 and σ(R) is the r.m.s. mass fluctuation within a top-hat smoothing of scale R.

Sheth & Tormen's multiplicity function is normalized, i.e,

``∫_-∞^∞ dlnν stMF(lnν) = 1``

This function is based on [Cosmology Routine Library (CRL)](https://wwwmpa.mpa-garching.mpg.de/~komatsu/crl/).
"""
function stMF(lnν::Real)
    A, p, q = 0.3222, 0.3, 0.707
    ν = exp(lnν)
    mf = A * (1 + (q * ν)^-p) * exp(-q * ν / 2) * √(q * ν / 2π)
end

"""
    jenkinsMF(lnν)

Jenkins et al.'s halo multiplicity function.

*Reference*: Equation (B3) of Jenkins et al., MNRAS, 321, 372 (2001)
- `jenkinsMF(lnν)` corresponds to f(σ)/2 in this reference, where σ=√(δc/ν).
- We use the fitting function for Δm = 180. Here, Δm is an overdensity within a spherical region of radius R, whose mean density is equal to Δm times the mean **mass** density of the Universe.

# Arguments
- `lnν::Real`: natural logarithm of a threshold, ν, i.e., `lnν` = log(ν), defined by ν ≡ [δc/σ(R)]^2. Here, δc = 1.6865 and σ(R) is the r.m.s. mass fluctuation within a top-hat smoothing of scale R.

Jenkins et al.'s multiplicity function is NOT normalized, i.e,

``∫_-∞^∞ dlnν jenkinsMF(lnν) ≠ 1``

This function is based on [Cosmology Routine Library (CRL)](https://wwwmpa.mpa-garching.mpg.de/~komatsu/crl/).
"""
function jenkinsMF(lnν::Real)
    ν = exp(lnν)
    σ = 1.6865 / √ν
    mf = 0.5 * 0.301 * exp(-abs(log(1 / σ) + 0.64)^3.82)
end


module postwidderTests

using Test
using InverseLaplace: postwid, _PWcoeffs

# Compare against results reported in Algorithm 368: Numerical inversion of Laplace transforms [D5].

#### F(s) = 1/sqrt(s), f(t) = 1/sqrt(π*t)

# test for single time point
setprecision(53)
t = 1.0
@test isapprox(postwid(s -> 1/sqrt(s), t, v = _PWcoeffs(18)), 1/sqrt(pi*t), rtol = 1e-4)
t = 5.0
@test isapprox(postwid(s -> 1/sqrt(s), t, v = _PWcoeffs(18)), 1/sqrt(pi*t), rtol = 1e-4)

setprecision(128)
t = 1.0
@test isapprox(postwid(s -> 1/sqrt(s), t, v = _PWcoeffs(36)), 1/sqrt(pi*t), rtol = 1e-12)
t = 5.0
@test isapprox(postwid(s -> 1/sqrt(s), t, v = _PWcoeffs(36)), 1/sqrt(pi*t), rtol = 1e-12)

# test across t array
t = 1.0:10.0

setprecision(53)
@test isapprox(postwid(s -> 1/sqrt(s), t, v = _PWcoeffs(18)), 1 ./ sqrt.(pi.*t), rtol = 1e-4)

setprecision(128)
@test isapprox(postwid(s -> 1/sqrt(s), t, v = _PWcoeffs(36)), 1 ./ sqrt.(pi.*t), rtol = 1e-12)

#### F(s) = 1/s^4, f(t) = t^3 / 6

# test for single time point
setprecision(53)
t = 1.0
@test isapprox(postwid(s -> 1/s^4, t, v = _PWcoeffs(18)), t^3/6, rtol = 1e-4)
t = 5.0
@test isapprox(postwid(s -> 1/s^4, t, v = _PWcoeffs(18)), t^3/6, rtol = 1e-4)

setprecision(128)
t = 1.0
@test isapprox(postwid(s -> 1/s^4, t, v = _PWcoeffs(36)), t^3/6, rtol = 1e-12)
t = 5.0
@test isapprox(postwid(s -> 1/s^4, t, v = _PWcoeffs(36)), t^3/6, rtol = 1e-12)

# test across t array
t = 1.0:10.0

setprecision(53)
@test isapprox(postwid(s -> 1/s^4, t, v = _PWcoeffs(18)), t.^3 ./6, rtol = 1e-4)

setprecision(128)
@test isapprox(postwid(s -> 1/s^4, t, v = _PWcoeffs(36)), t.^3 ./6, rtol = 1e-12)

#### F(s) = 1/(s+1), f(t) = exp(-t)
## for larger t you need a higher amount of coeffs depending on accuracy

# test for single time point
setprecision(53)
t = 1.0
@test isapprox(postwid(s -> 1/(s+1), t, v = _PWcoeffs(18)), exp(-t), rtol = 1e-4)
t = 5.0
@test isapprox(postwid(s -> 1/(s+1), t, v = _PWcoeffs(18)), exp(-t), rtol = 1e-3)

setprecision(128)
t = 1.0
@test isapprox(postwid(s -> 1/(s+1), t, v = _PWcoeffs(36)), exp(-t), rtol = 1e-12)
t = 5.0
@test isapprox(postwid(s -> 1/(s+1), t, v = _PWcoeffs(36)), exp(-t), rtol = 1e-8)

# test across t array
t = 1.0:10.0

setprecision(53)
@test isapprox(postwid(s -> 1/(s+1), t, v = _PWcoeffs(18)), exp.(-t), rtol = 1e-3)

setprecision(128)
@test isapprox(postwid(s -> 1/(s+1), t, v = _PWcoeffs(36)), exp.(-t), rtol = 1e-8)

end # module


using Plots

function bracket_minimum(f, x=0, s=1e-2, k=2.0)
    a, ya = x, f(x);
    b, yb = a + s, f(a + s);

    if yb > ya
        a, b = b, a;
        ya, yb = yb, ya;
        s = -s;
    end

    while true
        c, yc = b + s, f(b + s);
        if yc > yb
            return a < c ? (a, c) : (c, a);
        end

        a, ya, b, yb = b, yb, c, yc;
        s *= k;
    end
end

# test

f1 = x -> (1 .+ x).^2;
f2 = x -> (1 + x)^2;
x = -3:0.1:3;
y = f1(x);

a, b = bracket_minimum(f2, -2, 1e-2, 1.01);

plot(x, y)
vline!([a, b])


import RowEchelon
import LinearAlgebra

#=
Homework 8
Section 4.2
Problems 5
=#

A = [1 -4 0 2 0 ;
     0 0 1 -5 0 ;
     0 0 0 0 2]

nulA = LinearAlgebra.nullspace(A)
println("Original Matrix, A:")
println(A)
println("NulA:")
println(nulA)


"""
    angle(v::AbstractVector{<:Real}) -> Float

Angle the vector `v` makes with the x axis, throws error if vector is not of size 2
"""
function angle(v::AbstractVector{<:Real})
    if length(v)!=2 error("Vector must be of length 2") end
    atan(v[2],v[1])
end

angle(v::SVector{2, <:Real})  = atan(v[2],v[1])

"""
    sqrt(z, branchcutangle::Real=π) -> typeof(z)

Return \\sqrt{x}, analytically continued from real line with the branch cut from 0 in a straight line along `branchcutangle`
"""
function sqrt(z, branchcutangle::Real)

    if branchcutangle > 0
        if angle(z) > branchcutangle return -sqrt(z) end
    else
        if angle(z) < branchcutangle return -sqrt(z) end
    end

    return sqrt(z)
end

"""
    round(num::Real, base::Real) -> Real

Nearest multiple of `base` to `num`, result will be promoted to num and base
"""
round(num::Real, base::Real) = round(num/base)*base

"""
    ceil(num::Real, base::Real) -> Real

Smallest multiple of `base` greater than `num`, result will be promoted to num and base
"""
ceil(num::Real, base::Real) = ceil(num/base)*base

"""
    ceil(num::Real, base::Real) -> Real

Largest multiple of `base` smaller than `num`, result will be promoted to num and base
"""
floor(num::Real, base::Real) = floor(num/base)*base


# Generates the dataset to be used in GMM example for EM course

using Plots, Distributions, Random, DataFrames, CSV
Random.seed!(123)

# Set GMM parameters
n = 200
α = 0.4
μ₀ = [-1, 2]
μ₁ = [3.0, 7.0]
Σ₀ = [2.0 1.0; 1.0 2.0]
Σ₁ = [1.5 0; 0 0.5]
𝒩₀ = MvNormal(μ₀, Σ₀)
𝒩₁ = MvNormal(μ₁, Σ₁)
X₁ = zeros(n)
X₂ = zeros(n)

for i=1:n
    X₁[i],X₂[i] = rand() < α ? rand(𝒩₁) : rand(𝒩₀)
end

# Store in data frame
df = DataFrame(X1 = X₁, X2 = X₂)

# Save to csv
CSV.write("gmm_observations.csv", df)


# basisfunctions.jl

"""
    fourier(k::Int)

Implements the Fourier basis of functions ϕ_k, defined, if k is odd, by
ϕ_k(x)=sqrt(2)sin((k+1)π*x) and if k is even, by ϕ_k(x)=sqrt(2)cos(k*x).
This function is defined for k≥0 and ϕ_0≡1.

# Examples

```julia
x = -1.0:0.01:2.0
y = fourier(3).(x)
```
"""
function fourier(k::Int)
  k ≥ 0 || throw(AssertionError("k should be non-negative"))
  sqrttwo = sqrt(2.0)
  if k == 0
    return x -> 1.0
  elseif isodd(k)
    return x -> sqrttwo * sinpi(float(k+1)*x)
  else
    return x -> sqrttwo * cospi(float(k)*x)
  end
end

"""
    faberschauderone(x::Float64)

Implements the first Faber-Schauder function defined by 1-2x for 0≤x≤1/2 and
-1+2x for 0.5≤x≤1, and is 1-periodically extended to all x∈R.

#See also: faberschauder

#Warning

Note the difference between faberschauderone and faberschauder. The first is a
function that takes a Float64 and returns a Float64, the second takes (j,k) and
returns a anonymous function that takes a Float64 and returns a Float64.

#Examples

```julia
x=-2.0:0.001:2.0
y=faberschauderone.(x)
```
"""
function faberschauderone(x::Float64)
    y = mod(x, 1.0)
    if 0≤y≤0.5
        return 1.0 - 2*y
    else
        return -1.0 + 2*y
    end
end

"""
    faberschauder(j::Int, k::Int)

Implements the k-th Faber-Schauder function of level j. Here, j≥0 and 1≤k≤2^j.
It is a one periodic function and defined on [0,1] by 2^(j+1)(x-(k-1)2^(-j)) on
(k-1)2^(-j)≤x≤(k-1/2)2^(-j) and 1 - 2^(j+1)(x-(k-1/2)2^(-j)) on
[(k-1/2)2^(-j), k2^(-j)] and zero outside these intervals.

# See also: faberschauderone.

#Warning

Note the difference between faberschauderone and faberschauder. The first is a
function that takes a Float64 and returns a Float64, the second takes (j,k) and
returns a anonymous function that takes a Float64 and returns a Float64.

#Example using PyPlot
```julia
using PyPlot
clf()
j=3
x=-2.0:0.001:2.0
for k in 1:2^j
    y = faberschauder(j,k).(x)
    plot(x,y)
end
```
"""
function faberschauder(j::Int,k::Int)
  j ≥ 0 || throw(AssertionError("j should be a nonnegative integer."))
  1≤k≤2^j || throw(AssertionError("k should be an integer between 1 and 2^j."))
  twotothepowerminusj = 2.0^-j
  k_float = float(k)
  j_float = float(j)
  return function(x::Float64)
    y = mod(x,1.0)
    if y ≤ (k_float-1.0)*twotothepowerminusj || y ≥ k_float*twotothepowerminusj
      return 0.0
    elseif (k_float-1.0)*twotothepowerminusj < y ≤ (k_float-0.5)*twotothepowerminusj
      return 2.0^(j_float+1.0)*y-2*k_float+2.0
    else
      return 2.0*k_float - 2.0^(j_float+1.0) * y
    end
  end
end


### A Pluto.jl notebook ###
# v0.12.18

using Markdown
using InteractiveUtils

# This Pluto notebook uses @bind for interactivity. When running this notebook outside of Pluto, the following 'mock version' of @bind gives bound variables a default value (instead of an error).
macro bind(def, element)
    quote
        local el = $(esc(element))
        global $(esc(def)) = Core.applicable(Base.get, el) ? Base.get(el) : missing
        el
    end
end

# ╔═╡ 45f1331e-59b1-11eb-2bc0-c138b1421b55
begin
	using Pkg
	Pkg.activate(".")
	Pkg.instantiate()
end

# ╔═╡ fc1a9e20-56aa-11eb-29b1-3be8043c6ede
begin
	using Random
	Random.seed!(0)
	using Zygote
	using LinearAlgebra
	using Plots
	using PlutoUI
	using Images
	using LaTeXStrings
end

# ╔═╡ 60a23536-59d0-11eb-368a-5588b71a6093
md"# Curve fitting using AD"

# ╔═╡ 832e2c5a-59ce-11eb-32d7-85b05440f919
md"## Problem Formulation"

# ╔═╡ e35400cc-56ad-11eb-2794-592020c6ac7e
md"### The underlying mathematics of the problem"

# ╔═╡ 761fc420-57af-11eb-24b3-9728b06c482e
md"We are trying to fit a function $f_{org}(x)$ to a polynomial function $f_{ML}(x)$"

# ╔═╡ 021f6506-57b1-11eb-2179-7b6f153fce2f
md"$$f_{ML}(x_i) = \displaystyle\sum\limits_{j=0}^nw_jx_i^j = w_0 + x_iw_1 + x_i^2w_2+\ldots+x_i^nw_n$$"

# ╔═╡ df0e42cc-59be-11eb-367e-cf25ab5f105f
md"so that $f_{org}(x_i) \approx f_{ML}(x_i)$"

# ╔═╡ 2730398c-59bf-11eb-060e-276c0138395e
md"We define $X, \textbf{w}, \textbf{y}$ for a more concise and all-round understanding of the problem,"

# ╔═╡ e50c3cc4-59b7-11eb-1479-8bba33d251fd
md"$\begin{gather}
X=
\begin{bmatrix}
1 & x_1 & \ldots & x_1^n\\
\vdots & & \ddots &\vdots \\
1 & x_m & & x_m^n
\end{bmatrix}
\quad\quad\quad
\textbf{w} = \begin{bmatrix}
w_o \\
w_1 \\
\vdots\\
w_n
\end{bmatrix}
\quad\quad\quad
\textbf{y} =
\begin{bmatrix}
y_1 \\
y_2 \\
\vdots\\
y_m
\end{bmatrix}
\end{gather}$"

# ╔═╡ 51545508-59b9-11eb-3f94-59e743bfd1a8
md" where,
- The matrix $X$ is the data matrix. $X$ is a $(m\times (n+1)) matrix$. $m$ is the number of data points and $n$ is the n is the degree of the plyonomial. The choice of this polynomial becomes clear in the next step.
- The vector $\textbf{w}$ represents the weights vector. $w_0$ the bias. $w_j$ is the weight corresponding to $x^j$.
- The vector $\textbf{y}$ represents the results vector. $y_i$ is the result of $x_i$ on applying the function $f$"

# ╔═╡ 70b0cea0-59ba-11eb-2ae9-439069c94467
md"$\begin{gather}
\begin{bmatrix}
y_1 \\
y_2 \\
\vdots\\
y_m
\end{bmatrix}
=
\begin{bmatrix}
1 & x_1 & \ldots & x_1^n\\
\vdots & & \ddots &\vdots \\
1 & x_m & & x_m^n
\end{bmatrix}
\odot
\begin{bmatrix}
w_o \\
w_1 \\
\vdots\\
w_n
\end{bmatrix}
\end{gather}$"

# ╔═╡ ea14bfa4-59ba-11eb-35a1-6548f64bed7e
md"$\begin{gather}
\quad \quad \
\begin{bmatrix}
y_1 \\
y_2 \\
\vdots\\
y_m
\end{bmatrix}
=
\begin{bmatrix}
w_o + x_1\times w_1 &\ldots  &+x_1^n \times w_n\\
\vdots \\
w_0 + x_m\times w_1  &\ldots &+x_m^n\times w_n
\end{bmatrix}
\end{gather}$"

# ╔═╡ 32bb7c2c-59bc-11eb-0914-d99847d081a3
load(download("https://thehritikjain.files.wordpress.com/2020/05/1_e4_ptjctmaofsrpzczbv-g-1.jpeg?w=935"))

# ╔═╡ d2ea16ee-57b7-11eb-2d27-f3c04343dc59
md"Before we approach the real problem of solving the approximate function, we need a predicting function and loss function to evaluate good our approximation is."

# ╔═╡ fa7fe6b4-59c1-11eb-2486-d90418eaee4a
md"### Predict and loss functions : "

# ╔═╡ e8864a02-59c1-11eb-12ff-c9a4ffde849d
md"#### Predict function "

# ╔═╡ bf75963a-56ac-11eb-2b34-21b724a59f96
function predict(x, ω::Vector)
	n = length(ω)
	x_vec = [x^i for i= 0:n-1]
	return dot(x_vec, ω)
end

# ╔═╡ c8a25b68-59c1-11eb-363f-0772ec4edbf2
md"**Usage Example :**"

# ╔═╡ 10be275c-59c1-11eb-10dc-4becfaf27f8d
begin
	# Try changing the values
	local ω = rand(5) # 4 polynomial weight vector
	local x = 3. # input
	predict(x, ω)
end

# ╔═╡ d928b270-59c1-11eb-09d0-a9c3c878fe2d
md"#### Square Loss function "

# ╔═╡ 0129d404-57b8-11eb-3dde-efa8a687c1d4
function sq_loss(func, X, ω::Vector)
	n = length(ω)
	y = func.(X)
	ŷ = [predict(x, ω) for x in X]
	return sum((y-ŷ).^2)/n
end

# ╔═╡ bc758f7c-59c1-11eb-3f4a-5b987c7c6e0b
md"**Usage Example :**"

# ╔═╡ 55bd943c-59c1-11eb-19d3-a7bba6fcbd30
begin
	local ω = rand(5) # 4 polynomial weight vector
	local X = [3.] # input
	local loss = sq_loss(sin, X, ω)
end

# ╔═╡ 00f8133c-57be-11eb-19d0-33006b7bbb06
md"## Data Generation"

# ╔═╡ 6d407f0e-59ce-11eb-0ed6-33f817c914a3
md"Choose the number of data points : "

# ╔═╡ 4b28aee8-56ac-11eb-3be7-0de4e9cb211b
@bind n Slider(0:10, show_value = true, default=8)

# ╔═╡ e6d69724-57bf-11eb-0337-db41600a8ebc
md"We define a function that we want to approximate. **org\_func** is the original function. We are trying to approximate *sin(x)* here. You can try your own function that you want to approximat inside org\_func and chnages will be reflected in rest of the notebook."

# ╔═╡ 9801c9ca-57bf-11eb-3f30-93055ccd6a0f
function org_func(x)
	return sin(x)
end

# ╔═╡ 2fcaeb98-56b1-11eb-0198-a3a5088dc007
begin
	X = [x for x in range(randn(), 12, length = n)]
	y = org_func.(X) +0.1* randn((n))
	plot(-0.5:0.01:12, org_func.(-0.5:0.01:12), label = L"sin(x)")
	scatter!(X, y, label = "data points", title = "Our Data points")
end

# ╔═╡ bd151b36-56b1-11eb-03d1-094237a3bafa
md"## Approximation"

# ╔═╡ c7c2e3f2-57be-11eb-15c0-dbe72c95d654
md"We will try to solve the problem of approximating the function by two differnt ways:
- Linear Algebra Methods
- Gradient Descent using AD"

# ╔═╡ 377b50e4-57bf-11eb-1741-a73fda781579
md" ### Approximation by Linear Algebra"

# ╔═╡ 50c00e44-59c4-11eb-2726-27bee50e4fdd
md"$\begin{gather}
\begin{bmatrix}
y_1 \\
y_2 \\
\vdots\\
y_m
\end{bmatrix}
=
\begin{bmatrix}
1 & x_1 & \ldots & x_1^n\\
\vdots & & \ddots &\vdots \\
1 & x_m & & x_m^n
\end{bmatrix}
\odot
\begin{bmatrix}
w_o \\
w_1 \\
\vdots\\
w_n
\end{bmatrix}
\end{gather}$"

# ╔═╡ 586240fe-59c4-11eb-1427-312c47374f8a
md"$
\ \ \ \
\begin{gather}
\begin{bmatrix}
w_o \\
w_1 \\
\vdots\\
w_n
\end{bmatrix}
=
\begin{bmatrix}
1 & x_1 & \ldots & x_1^n\\
\vdots & & \ddots &\vdots \\
1 & x_m & & x_m^n
\end{bmatrix}^{-1}
\odot
\begin{bmatrix}
y_1 \\
y_2 \\
\vdots\\
y_m
\end{bmatrix}

\end{gather}$"

# ╔═╡ 4f5c808e-57bf-11eb-2a7a-3995d6f8f866
md"This may seem like a very viable method, but try considering the case where $X$ is invertible. It is left as an exercise to the reader."

# ╔═╡ 5dd40e02-57bf-11eb-2764-0f3f2b95517b
md" ### Approximation by Gradient Descent"

# ╔═╡ ea205094-59c9-11eb-0f04-818f051b7d1e
md"We will be using gradient descent algorithm to find $\textbf{w}$ here. We are tired of mathematics by now and we somehow want the differentaition to happen automatically(almost magically). We will be using **Automatic Differentation**(or a more intuitive name : **Algorthimic Differentiation**). While how automatic differintation works is another day's agenda, we can have a little taste of magic of what AD does for us."

# ╔═╡ b13969a4-59ca-11eb-15d8-0f9b5f225fed
md"The gradient of a function $f(x)$ at $x_0$ is defined as $\displaystyle\frac{df(x)}{dx}\Bigg|_{x=x_0}$

$f^\prime(x_0) = \displaystyle\frac{df(x)}{dx}\Bigg|_{x=x_0}$"

# ╔═╡ e4f40500-59cb-11eb-1020-3b345469fb1f
md"The Zygote package gives us a *gradient* API that we can use to find gradients"

# ╔═╡ 88002972-59cc-11eb-3d75-ab7e7bb45954
md"$f(x) = x^2+4x+1$
$f^\prime(x) = 2x+4$
$f^\prime(5) = 2\times 5 + 4 = 14$"

# ╔═╡ 3255da08-59cc-11eb-0bdc-61a05344ad50
gradient(x->x^2+4x+1,  5)

# ╔═╡ cd6a28f8-59cc-11eb-28f8-35c3a04414d8
md"$f(x, y) = x^2+y^2+3xy$
$\displaystyle\frac{\partial f(x,y)}{\partial x} = 2x+3y$
$\displaystyle\frac{\partial f(x,y)}{\partial y} = 2y+3x$"

# ╔═╡ 4506b2d0-59cc-11eb-22e2-5f43abeb3ccd
gradient( (x,y) -> x^2 + y^2 + 3x*y, 2,3)

# ╔═╡ 608cc19c-59cd-11eb-092d-558d322966c6
md"A train step function that returns the updated weights vector $\textbf{w}$"

# ╔═╡ 84566232-57bf-11eb-390b-f33370fbd9a3
function train_step(func, X, ω, η)::typeof(ω)
	grad = gradient(ω->sq_loss(func, X, ω), ω)[1]
	ω -= η * grad
	return ω
end

# ╔═╡ de0aeb62-59cd-11eb-3e5f-a7cb176093c9
md"Choose the degree of polynomial of  $f_{ML}(x)$. You may need to choose different $\eta$ and number of epochs to account for the model change"

# ╔═╡ ae1c0e18-59cd-11eb-2e61-fbe810bd5fa0
@bind n_model Slider(1:1:7, default = 4, show_value = true)

# ╔═╡ 5c85511c-5811-11eb-0f2e-2352309f2261
## Intializing weights
ω = rand(n_model+1)

# ╔═╡ 11337388-57c1-11eb-3418-3979d91e6af2
function train(func, X, ω, η, epochs)::typeof(ω)
	for i = 1:1:epochs
		ω =  train_step(func, X, ω, η)
		loss = sq_loss(func, X, ω)
		if isnan(loss) || isinf(loss) || iszero(loss)
			throw(DomainError("Loss cannot take $(loss) value"))
		end
	end
	return ω
end

# ╔═╡ d722598c-57c2-11eb-3452-531fb6522e34
ω_trained = train(org_func, X, ω, 7e-9, 1e6)

# ╔═╡ 11d38860-59c6-11eb-06e2-b10697f21f1d
md"It is upto the user to decide how much accuracy he wants given the time constraints"

# ╔═╡ 8b6bc65e-57c7-11eb-372f-bd6f700363e1
loss = sq_loss(org_func, X, ω_trained)

# ╔═╡ 328c4584-57c4-11eb-3cf5-a57e033cf574
pred = [predict(x, ω_trained) for x in -0.5:0.01:13]

# ╔═╡ a0dfe352-57c3-11eb-348f-8380ee6b3caf
begin
	plot(-0.5:0.01:13, org_func.(-0.5:0.01:13), label = L"f_{org}(x)")
	scatter!(X, y, label = "Orginal data points")
	ŷ = [predict(x, ω_trained) for x in X]
	scatter!(X, ŷ, label = "Predicted data points")
	plot!(-0.5:0.01:13, pred, label = L"f_{ML}(x)")
end

# ╔═╡ Cell order:
# ╟─45f1331e-59b1-11eb-2bc0-c138b1421b55
# ╟─fc1a9e20-56aa-11eb-29b1-3be8043c6ede
# ╟─60a23536-59d0-11eb-368a-5588b71a6093
# ╟─832e2c5a-59ce-11eb-32d7-85b05440f919
# ╟─e35400cc-56ad-11eb-2794-592020c6ac7e
# ╟─761fc420-57af-11eb-24b3-9728b06c482e
# ╟─021f6506-57b1-11eb-2179-7b6f153fce2f
# ╟─df0e42cc-59be-11eb-367e-cf25ab5f105f
# ╟─2730398c-59bf-11eb-060e-276c0138395e
# ╟─e50c3cc4-59b7-11eb-1479-8bba33d251fd
# ╟─51545508-59b9-11eb-3f94-59e743bfd1a8
# ╟─70b0cea0-59ba-11eb-2ae9-439069c94467
# ╟─ea14bfa4-59ba-11eb-35a1-6548f64bed7e
# ╟─32bb7c2c-59bc-11eb-0914-d99847d081a3
# ╟─d2ea16ee-57b7-11eb-2d27-f3c04343dc59
# ╟─fa7fe6b4-59c1-11eb-2486-d90418eaee4a
# ╟─e8864a02-59c1-11eb-12ff-c9a4ffde849d
# ╠═bf75963a-56ac-11eb-2b34-21b724a59f96
# ╟─c8a25b68-59c1-11eb-363f-0772ec4edbf2
# ╠═10be275c-59c1-11eb-10dc-4becfaf27f8d
# ╟─d928b270-59c1-11eb-09d0-a9c3c878fe2d
# ╠═0129d404-57b8-11eb-3dde-efa8a687c1d4
# ╟─bc758f7c-59c1-11eb-3f4a-5b987c7c6e0b
# ╠═55bd943c-59c1-11eb-19d3-a7bba6fcbd30
# ╟─00f8133c-57be-11eb-19d0-33006b7bbb06
# ╟─6d407f0e-59ce-11eb-0ed6-33f817c914a3
# ╟─4b28aee8-56ac-11eb-3be7-0de4e9cb211b
# ╟─e6d69724-57bf-11eb-0337-db41600a8ebc
# ╠═9801c9ca-57bf-11eb-3f30-93055ccd6a0f
# ╠═2fcaeb98-56b1-11eb-0198-a3a5088dc007
# ╟─bd151b36-56b1-11eb-03d1-094237a3bafa
# ╟─c7c2e3f2-57be-11eb-15c0-dbe72c95d654
# ╟─377b50e4-57bf-11eb-1741-a73fda781579
# ╟─50c00e44-59c4-11eb-2726-27bee50e4fdd
# ╟─586240fe-59c4-11eb-1427-312c47374f8a
# ╟─4f5c808e-57bf-11eb-2a7a-3995d6f8f866
# ╟─5dd40e02-57bf-11eb-2764-0f3f2b95517b
# ╟─ea205094-59c9-11eb-0f04-818f051b7d1e
# ╟─b13969a4-59ca-11eb-15d8-0f9b5f225fed
# ╟─e4f40500-59cb-11eb-1020-3b345469fb1f
# ╟─88002972-59cc-11eb-3d75-ab7e7bb45954
# ╠═3255da08-59cc-11eb-0bdc-61a05344ad50
# ╟─cd6a28f8-59cc-11eb-28f8-35c3a04414d8
# ╠═4506b2d0-59cc-11eb-22e2-5f43abeb3ccd
# ╟─608cc19c-59cd-11eb-092d-558d322966c6
# ╠═84566232-57bf-11eb-390b-f33370fbd9a3
# ╟─de0aeb62-59cd-11eb-3e5f-a7cb176093c9
# ╟─ae1c0e18-59cd-11eb-2e61-fbe810bd5fa0
# ╠═5c85511c-5811-11eb-0f2e-2352309f2261
# ╠═11337388-57c1-11eb-3418-3979d91e6af2
# ╠═d722598c-57c2-11eb-3452-531fb6522e34
# ╟─11d38860-59c6-11eb-06e2-b10697f21f1d
# ╠═8b6bc65e-57c7-11eb-372f-bd6f700363e1
# ╠═328c4584-57c4-11eb-3cf5-a57e033cf574
# ╟─a0dfe352-57c3-11eb-348f-8380ee6b3caf