text:
"""
	struct Quaternion

``Unit quaternions`` provide a convenient mathematical notation for representing ``orientations`` and ``rotations`` of objects in three dimensions.
See ``https://github.com/JuliaGeometry/Quaternions.jl/blob/master/src/Quaternion.jl``

# Example
See ``https://en.wikipedia.org/wiki/Quaternions_and_spatial_rotation``
"""
struct Quaternion

	w::Float64
	x::Float64
	y::Float64
	z::Float64

	# constructor
	function Quaternion()
		new(1,0,0,0)
	end

	# constructor
	function Quaternion(w::Float64,x::Float64,y::Float64,z::Float64)
		norm=sqrt(w * w + x * x + y * y + z * z)
		w/=norm
		x/=norm
		y/=norm
		z/=norm
		new(w,x,y,z)
	end

	# constructor
	function Quaternion(axis::Point3d,angle::Float64)
		axis=normalized(axis)
		c = cos(0.5*angle)
		s = sin(0.5*angle)
		new(c,s*axis[1],s*axis[2],s*axis[3])
	end

end

(*)(q::Quaternion, w::Quaternion) = Quaternion(
	q.w * w.w - q.x * w.x - q.y * w.y - q.z * w.z,
	q.w * w.x + q.x * w.w + q.y * w.z - q.z * w.y,
	q.w * w.y - q.x * w.z + q.y * w.w + q.z * w.x,
	q.w * w.z + q.x * w.y - q.y * w.x + q.z * w.w)

"""
	convertToQuaternion(T::Matrix4)

Transform the affine matrix `T::Matrix4` to quaternion.

# Example
`Euler Angles` to `Quaternion` Conversion.
By combining the quaternion representations of the Euler rotations we get for the Body 3-2-1 sequence, where the airplane first does ``yaw`` (Body-Z) turn during taxiing onto the runway, then ``pitches`` (Body-Y) during take-off, and finally ``rolls`` (Body-X) in the air.
```
 julia> Rx(a) =
 [1.0     0.0     0.0  0.0;
  0.0  cos(a) -sin(a)  0.0;
  0.0  sin(a)  cos(a)  0.0;
  0.0     0.0     0.0  1.0];

 julia> Ry(b) =
 [cos(a)  0.0  sin(a)  0.0;
  0.0     1.0     0.0  0.0;
 -sin(a)  0.0  cos(a)  0.0;
  0.0     0.0     0.0  1.0];

 julia> Rz(c) =
 [cos(a) -sin(a)  0.0  0.0;
  sin(a)  cos(a)  0.0  0.0;
 	 0.0    0.0   1.0  0.0;
 	 0.0    0.0   0.0  1.0];

 julia> T = GL.Matrix4( Rz(c) * Ry(b) * Rx(a) );
 4×4 MArray{Tuple{4,4},Float64,2,16}:
   0.75      -0.216506   0.625     0.0
   0.433013   0.875     -0.216506  0.0
  -0.5        0.433013   0.75      0.0
   0.0        0.0        0.0       1.0

 julia> q = GL.convertToQuaternion(T::GL.Matrix4)
 ViewerGL.Quaternion(0.9185586535436918, 0.1767766952966369, 0.30618621784789724, 0.1767766952966369)
```
"""
function convertToQuaternion(T::Matrix4)
	# See https://arxiv.org/pdf/math/0701759.pdf
	a2 = 1.0 + T[1,1] + T[2,2] + T[3,3]
	b2 = 1.0 + T[1,1] - T[2,2] - T[3,3]
	c2 = 1.0 - T[1,1] + T[2,2] - T[3,3]
	d2 = 1.0 - T[1,1] - T[2,2] + T[3,3]

	if a2 >= max(b2, c2, d2)
		a = sqrt(a2)/2
		return Quaternion(a, (T[3,2]-T[2,3])/4a, (T[1,3]-T[3,1])/4a, (T[2,1]-T[1,2])/4a)
	elseif b2 >= max(c2, d2)
		b = sqrt(b2)/2
		return Quaternion((T[3,2]-T[2,3])/4b, b, (T[2,1]+T[1,2])/4b, (T[1,3]+T[3,1])/4b)
	elseif c2 >= d2
		c = sqrt(c2)/2
		return Quaternion((T[1,3]-T[3,1])/4c, (T[2,1]+T[1,2])/4c, c, (T[3,2]+T[2,3])/4c)
	else
		d = sqrt(d2)/2
		return Quaternion((T[2,1]-T[1,2])/4d, (T[1,3]+T[3,1])/4d, (T[3,2]+T[2,3])/4d, d)
	end
end

"""
	convertToMatrix(q::Quaternion)

Quaternion to affine matrix `T::Matrix4` transformation.
# Example
```
julia> a,b,c = pi/6,pi/3,pi/24
(0.5235987755982988, 1.0471975511965976, 0.1308996938995747)

julia> T = GL.Matrix4( Rz(c) * Ry(b) * Rx(a) )
4×4 MArray{Tuple{4,4},Float64,2,16}:
  0.75      -0.216506   0.625     0.0
  0.433013   0.875     -0.216506  0.0
 -0.5        0.433013   0.75      0.0
  0.0        0.0        0.0       1.0

julia> q = GL.convertToQuaternion(T::GL.Matrix4)
ViewerGL.Quaternion(0.9185586535436918, 0.1767766952966369, 0.30618621784789724, 0.1767766952966369)

julia> GL.convertToMatrix(q::GL.Quaternion)
4×4 MArray{Tuple{4,4},Float64,2,16}:
  0.75      -0.216506   0.625     0.0
  0.433013   0.875     -0.216506  0.0
 -0.5        0.433013   0.75      0.0
  0.0        0.0        0.0       1.0
```
"""
function convertToMatrix(q::Quaternion)
	sx, sy, sz = 2q.w * q.x, 2q.w * q.y, 2q.w * q.z
	xx, xy, xz = 2q.x^2, 2q.x * q.y, 2q.x * q.z
	yy, yz, zz = 2q.y^2, 2q.y * q.z, 2q.z^2
	return Matrix4(
		1 - (yy + zz),      xy - sz,       xz + sy, 0.0,
		     xy + sz ,1 - (xx + zz),       yz - sx, 0.0,
		     xz - sy ,      yz + sx, 1 - (xx + yy), 0.0,
		0.0,0.0,0.0,1.0)
end


text:
using Documenter, LibASICamera

makedocs(sitename="LibASICamera API")

deploydocs(repo = "github.com/AlfTetzlaff/LibASICamera.jl.git")

ext: jl


text:
importall MathProgBase.SolverInterface

###############################################################################
# Solver objects
export KnitroSolver
immutable KnitroSolver <: AbstractMathProgSolver
    options
end
KnitroSolver(;kwargs...) = KnitroSolver(kwargs)

type KnitroMathProgModel <: AbstractNonlinearModel
    options
    inner::KnitroProblem

    numVar::Int
    numConstr::Int
    nnzJ::Int
    nnzH::Int

    varLB::Vector{Float64}
    varUB::Vector{Float64}
    constrLB::Vector{Float64}
    constrUB::Vector{Float64}

    jac_con::Vector{Int32}
    jac_var::Vector{Int32}
    hess_row::Vector{Int32}
    hess_col::Vector{Int32}

    varType::Vector{Int32}
    objType::Int32
    objFnType::Int32
    constrType::Vector{Int32}
    constrFnType::Vector{Int32}

    initial_x
    sense::Int32
    d::AbstractNLPEvaluator

    hasbeenrestarted::Bool

    solve_time::Float64

    function KnitroMathProgModel(;options...)
        new(options)
    end
end

NonlinearModel(s::KnitroSolver) = KnitroMathProgModel(;s.options...)
LinearQuadraticModel(s::KnitroSolver) = NonlinearToLPQPBridge(NonlinearModel(s))

###############################################################################
# Begin interface implementation

function sparse_merge_hess_duplicates(I, J, m, n)
    V = [Int[i] for i in 1:length(I)]
    for i in 1:length(I) # make upper triangular
        if I[i] > J[i]
            I[i],J[i] = J[i],I[i]
        end
    end
    findnz(sparse(I,J,V,m,n,vcat))
end

function sparse_merge_jac_duplicates(I, J, m, n)
    V = [Int[i] for i in 1:length(I)]
    findnz(sparse(I,J,V,m,n,vcat))
end

# generic nonlinear interface
function loadproblem!(m::KnitroMathProgModel,
                      numVar::Int,
                      numConstr::Int,
                      x_l, x_u, g_lb, g_ub,
                      sense::Symbol,
                      d::AbstractNLPEvaluator)
    features = features_available(d)
    has_hessian = (:Hess in features)
    if has_hessian
        initialize(d, [:Grad, :Jac, :Hess])
        Ihess, Jhess = hesslag_structure(d)
    else
        initialize(d, [:Grad, :Jac])
        Ihess = Int[]
        Jhess = Int[]
    end

    Ijac, Jjac = jac_structure(d)
    m.nnzJ = length(Ijac)
    m.nnzH = length(Ihess)
    jac_tmp = Array(Float64, m.nnzJ)
    hess_tmp = Array(Float64, m.nnzH)
    @assert length(Ijac) == length(Jjac)
    @assert length(Ihess) == length(Jhess)
    m.jac_con, m.jac_var, jac_indices = sparse_merge_jac_duplicates(Ijac, Jjac,
                                                                numConstr,
                                                                numVar)
    m.jac_con = m.jac_con .- 1
    m.jac_var = m.jac_var .- 1
    m.hess_row, m.hess_col, hess_indices = sparse_merge_hess_duplicates(Ihess,
                                                                    Jhess,
                                                                    numVar,
                                                                    numVar)
    m.hess_row = m.hess_row .- 1
    m.hess_col = m.hess_col .- 1
    n_jac_indices = length(jac_indices)
    n_hess_indices = length(hess_indices)

    m.varLB = x_l
    m.varUB = x_u
    m.constrLB = g_lb
    m.constrUB = g_ub
    m.numVar = length(m.varLB)
    m.numConstr = length(m.constrLB)
    @assert m.numVar == length(m.varUB)
    @assert m.numConstr == length(m.constrUB)

    for i in 1:m.numVar
        if m.varLB[i] == -Inf
            m.varLB[i] = -KTR_INFBOUND
        end
        if m.varUB[i] == Inf
            m.varUB[i] = KTR_INFBOUND
        end
    end

    for i in 1:m.numConstr
        if m.constrLB[i] == -Inf
            m.constrLB[i] = -KTR_INFBOUND
        end
        if m.constrUB[i] == Inf
            m.constrUB[i] = KTR_INFBOUND
        end
    end
    m.initial_x = C_NULL

    @assert sense == :Min || sense == :Max
    m.sense = (sense == :Min) ? KTR_OBJGOAL_MINIMIZE : KTR_OBJGOAL_MAXIMIZE
    # allow for the possibility of specializing to LINEAR or QUADRATIC?
    if isobjlinear(d)
        m.objType = KTR_OBJTYPE_LINEAR
        m.objFnType = KTR_FNTYPE_CONVEX
    elseif isobjquadratic(d)
        m.objType = KTR_OBJTYPE_QUADRATIC
        m.objFnType = KTR_FNTYPE_UNCERTAIN
    else
        m.objType = KTR_OBJTYPE_GENERAL
        m.objFnType = KTR_FNTYPE_UNCERTAIN
    end

    m.constrType = fill(KTR_CONTYPE_GENERAL, m.numConstr)
    m.constrFnType = fill(KTR_FNTYPE_UNCERTAIN, m.numConstr)
    for i=1:m.numConstr
        if isconstrlinear(d, i)
            # Bug in Knitro, this is unsafe until 10.1.2
            # See https://groups.google.com/d/msg/knitro/H9ck55HjY0s/U0r2-f6zAwAJ
            #m.constrType[i] = KTR_CONTYPE_LINEAR
            m.constrFnType[i] = KTR_FNTYPE_CONVEX
        end
    end

    m.varType = fill(KTR_VARTYPE_CONTINUOUS, m.numVar)

    # Objective callback
    eval_f_cb(x) = eval_f(d,x)

    # Objective gradient callback
    eval_grad_f_cb(x, grad_f) = eval_grad_f(d, grad_f, x)

    # Constraint value callback
    eval_g_cb(x, g) = eval_g(d, g, x)

    # Jacobian callback
    function eval_jac_g_cb(x, jac)
        eval_jac_g(d, jac_tmp, x)
        for i in 1:n_jac_indices
            jac[i] = sum(jac_tmp[jac_indices[i]])
        end
    end

    # Hessian callback
    function eval_h_cb(x, lambda, sigma, hess)
        eval_hesslag(d, hess_tmp, x, sigma, lambda)
        for i in 1:n_hess_indices
            hess[i] = sum(hess_tmp[hess_indices[i]])
        end
    end

    # Hessian-vector callback
    eval_hv_cb(x, lambda, sigma, hv) = eval_hesslag_prod(d, hv, x, sigma,
                                                         lambda)

    m.inner = createProblem()
    defined_hessopt = false; hessopt_value = 0
    for (param,value) in m.options
        param = string(param)
        if param == "KTR_PARAM_HESSOPT"
            defined_hessopt = true; hessopt_value = value
        end
        if param == "options_file"
            loadOptionsFile(m.inner, value)
        elseif param == "tuner_file"
            loadTunerFile(m.inner, value)
        else
            if haskey(paramName2Indx, param) # KTR_PARAM_*
                setOption(m.inner, paramName2Indx[param], value)
            else # string name
                setOption(m.inner, param, value)
            end
        end
    end

    # check and define default hessian option
    if (!has_hessian && !defined_hessopt) || (!has_hessian && !in(hessopt_value,2:6))
        setOption(m.inner,paramName2Indx["KTR_PARAM_HESSOPT"],6)
    end

    setCallbacks(m.inner, eval_f_cb, eval_g_cb, eval_grad_f_cb, eval_jac_g_cb,
                 eval_h_cb, eval_hv_cb)
end

getsense(m::KnitroMathProgModel) = m.sense
numvar(m::KnitroMathProgModel) = m.numVar
numconstr(m::KnitroMathProgModel) = m.numConstr

function optimize!(m::KnitroMathProgModel)
    if applicationReturnStatus(m.inner) == :Uninitialized
        if all(x->x==KTR_VARTYPE_CONTINUOUS, m.varType)
            initializeProblem(m.inner, m.sense, m.objType, m.varLB, m.varUB,
                              m.constrType, m.constrLB, m.constrUB, m.jac_var,
                              m.jac_con, m.hess_row, m.hess_col;
                              initial_x = m.initial_x)
        else
            initializeProblem(m.inner, m.sense, m.objType, m.objFnType,
                              m.varType, m.varLB, m.varUB, m.constrType,
                              m.constrFnType, m.constrLB, m.constrUB,
                              m.jac_var, m.jac_con, m.hess_row, m.hess_col;
                              initial_x = m.initial_x)
        end
    elseif !m.hasbeenrestarted
        restartProblem(m.inner)
    end
    t = time()
    solveProblem(m.inner)
    m.solve_time = time() - t
    m.hasbeenrestarted = false
end

function status(m::KnitroMathProgModel)
    applicationReturnStatus(m.inner)
end

getobjval(m::KnitroMathProgModel) = m.inner.obj_val[1]
getobjbound(m::KnitroMathProgModel) = get_mip_relaxation_bnd(m.inner)
getsolution(m::KnitroMathProgModel) = m.inner.x
getconstrsolution(m::KnitroMathProgModel) = m.inner.g
getreducedcosts(m::KnitroMathProgModel) =
    -1 .* m.inner.lambda[m.numConstr + 1:end]
getconstrduals(m::KnitroMathProgModel) = -1 .* m.inner.lambda[1:m.numConstr]
getrawsolver(m::KnitroMathProgModel) = m.inner
getsolvetime(m::KnitroMathProgModel) = m.solve_time

function warmstart(m::KnitroMathProgModel, x)
    m.initial_x = [Float64(i) for i in x]
    if applicationReturnStatus(m.inner) != :Uninitialized
        restartProblem(m.inner, m.initial_x, m.inner.lambda)
        m.hasbeenrestarted = true
    end
end

setwarmstart!(m::KnitroMathProgModel, x) = warmstart(m,x)
setvartype!(m::KnitroMathProgModel, typ::Vector{Symbol}) =
    (m.varType = map(t->rev_var_type_map[t], typ))

freemodel!(m::KnitroMathProgModel) = freeProblem(m.inner)


text:
using MiscUtil
using Test

@testset "MiscUtil.jl" begin

    ###########
    # sigmoid #
    ###########

     for x in [1e-6;0.0;1.0;5.0]
       @test abs(invsigmoid(sigmoid(x)) - x) < 1e-8
     end

    ####################
    # transformbetween #
    ####################

    for x in [1e-6;0.0;1.0;5.0]
        @test abs(invtransformbetween(transformbetween(x, -1.3, 10.1), -1.3, 10.1) - x) < 1e-8
    end

    ################
    # makepositive #
    ################

    for x in [1e-3;0.0;1.0;5.0]
        @test abs(invmakepositive(makepositive(x)) - x) < 1e-8
    end

end


text:
macro err_check(expr)
    quote
        ret = $(esc(expr))
        ret == 0 || mbed_err(ret)
        ret
    end
end

struct MbedException <: Exception
    ret::Cint
end

function show(io::IO, err::MbedException)
    print(io, "MbedTLS error code $(err.ret): $(strerror(err.ret))")
end

mbed_err(ret) = throw(MbedException(ret))

function strerror(ret, bufsize=1000)
    buf = Vector{UInt8}(undef, bufsize)
    ccall((:mbedtls_strerror, libmbedcrypto), Cint,
        (Cint, Ptr{Cvoid}, Csize_t),
        ret, buf, bufsize)
    s = unsafe_string(pointer(buf))
    if ret == MBEDTLS_ERR_SSL_FATAL_ALERT_MESSAGE
        s *= " (You may need to enable `ssl_conf_renegotiation!`. See " *
        "https://github.com/JuliaWeb/HTTP.jl/issues/342#issuecomment-432921180)"
    end
    return s
end


text:
using LinearAlgebra
#using Arpack # disabled. Use ArnoldMethod instead
using ..NEPCore, ..NEPTypes
import ..NEPCore.compute_Mder;
import ..NEPCore.compute_Mlincomb;
import Base.size;
export broyden;

function broyden_default_errmeasure(λ,v,r)
    return norm(r)/norm(v)
end

# The internal Broyden method (without deflation)
# which corresponds to version T in the paper below
function broyden_T(::Type{TT},nep::NEP;
                   v1=0,u1=[],λ1=TT(0),
                   CH=0,T1=0,W1=0,
                   maxit=100,
                   S=zeros(TT,0,0),X=zeros(TT,size(nep,1),0),
                   check_error_every=10,
                   print_error_every=1,
                   tol=1e-12,
                   threshold=0.4,
                   time0=time_ns(),
                   errmeasure::Function=broyden_default_errmeasure,
                   logger=0) where {TT<:Number}

    @parse_logger_param!(logger)

    # Check types are consistent

    v=Array{TT,1}(v1);
    u=Array{TT,1}(u1);
    X=Array{TT,2}(X);
    S=Array{TT,2}(S);
    CH=Array{TT,2}(CH);
    λ=TT(λ1);
    #x=vcat(v,λ);
    #x=Array{TT,1}(x);
    n=size(nep,1);
    p=size(S,1);

    rk = compute_Mlincomb(nep, λ, v+(X*((λ*Matrix{TT}(I, p, p) - S) \ u)))

    T = Matrix{TT}(T1)
    Wext = Matrix{TT}(undef, n, p+2)
    W = view(Wext, 1:n, 1:p+1)
    W[:,:] = Matrix{TT}(W1)
    errhist = Vector{real(TT)}(NaN * ones(real(TT), maxit))

    timehist = Vector{Float64}(undef, maxit) * NaN
    II = Matrix{TT}(I, p, p)

    Z = Matrix{TT}(undef, n, p+1)
    Tztilde = Vector{TT}(undef, n)
    aH = Vector{TT}(undef, p)'
    Wold = copy(W)
    ztilde = Vector{TT}(undef, n)
    waH = Vector{TT}(undef, p)'
    Z=T*W;
    for j=1:maxit

        Trk=T*rk;

        duλ=-(CH*Z)\(CH*Trk) # contains [Δu;Δλ]

        # Compute u and λ updates
        Δu=duλ[1:p];
        Δλ=duλ[end];
        # Compute v-update
        Δv=-Z*duλ-Trk

        γ=TT(1.0);
        tt=real(TT)(sqrt(abs(Δλ)^2+norm(Δv)^2));
        if (tt>threshold)  # Avoid too big λ-steps
            γ=TT(threshold)/tt;
        end

        v0=v; u0=u; λ0=λ;

        v=v+γ*Δv;
        u=u+γ*Δu;
        λ=λ+γ*Δλ;

        # Update Jacobian approximation
        vv=v+(X*((λ*II-S)\u));
        rkp::AbstractVector=compute_Mlincomb(nep,λ,vv);

        ztilde=(rkp-(1-γ)*rk)/γ;
        Tztilde=T*ztilde;

        # For w-update
        bH=[Δu' (Δλ)']/(norm(Δv)^2+norm(Δu)^2+norm(Δλ)^2);
        # For T-update
        β=norm(Δv)^2+norm(Δu)^2+abs(Δλ)^2+Δv'*Tztilde;
        aH=-(Δv'*T)/β

        # Update Z
        Z=Z+Tztilde*(aH*W+(1+aH*ztilde)*bH);

        # Update W
        W=W+ztilde*bH;

        # Update T
        T=T+((Tztilde)*aH);

        if (false) # Extra type check
            for q=[Z,T,W,Trk,duλ,Δu,Δv,λ,Δλ,Tztilde,T,aH,complex(β)]
                if (TT != eltype(q))
                    error("Bad type ",q);
                end

            end
        end

        # step forward
        rk=rkp;

        if (mod(j,check_error_every)==0)
            errhist[j]=errmeasure(λ,v+(X/(λ*II-S))*u,rk);
            timehist[j]=Float64((time_ns()-time0)*1e-9);
            if (mod(j,print_error_every)==0)
                d = opnorm(CH*v - reverse(Matrix{TT}(I, 1+p, 1), dims = 1))
            end

            push_iteration_info!(logger,j,err=errhist[j],λ=λ);
            if (errhist[j]<tol)
                return (λ,v,u,T,W,j,errhist[1:j],timehist[1:j])
            end
        end

    end

    push_info!(logger,"Too many iterations"); #" resnorm=",opnorm(rk));
    return (λ,v,u,T,W,maxit,errhist,timehist)

end

##################
"""
    S,V = broyden([eltype,]nep::NEP[,approxnep];kwargs)

Runs Broydens method (with deflation) for the nonlinear eigenvalue
problem defined by nep.
An approximate nep can be provided which is used as an
initialization of starting
matrix/vectors. The optional argument `approxnep` determines how
 to initiate the
algorithm. It can be an `NEP`, the symbol `:eye`
corresponding to starting
with an identity matrix, and a `Matrix` (of size ``n\times n``).
Beside most of the standard kwargs as described in [`augnewton`](@ref),
it supports `pmax` which is subspace used in deflation, essentially
the number of eigenvalues, `add_nans::Bool`  which
determines if `NaNs` should be added in book keeping.
`eigmethod` which can be `:eig`, `:eigs` or
`:invpow`. The `:invpow` is an implementation of the power method, which
is slow but works well e.g. for `BigFloat`. The kwarg `recompute_U`
determines if the `U`-matrix should be recomputed in every
deflation (which can be more robust). The implementation
has two loggers `logger` and `inner_logger`. The `logger`
corresponds to outer iterations (deflation)
and  `inner_logger` is the iterates in Broydens method.
The kwarg `check_error_every` and `print_error_every`
  detemine how often errors should be check and how
often they should be printed. For real problems
with complex conjugate symmetry, you may want
to set the kwarg `addconj=true` in order
to reduce computation by automatically adding the
complex conjugate vectors.

The method computes an invariant pair and can therefore find
several eigenvalues. The
retured value is (S,V) is an invariant pair and
the eigenvalues are on the diagonal of S.
Eigenpairs can be directly extracted with
[`get_deflated_eigpairs`](@ref).

# Example

```julia-repl
julia> nep=nep_gallery("dep0");
julia> S,V=broyden(nep);
julia> λ=S[1,1]
-0.15955391823299253 - 3.874865266487398e-19im
julia> minimum(svdvals(compute_Mder(nep,λ)))
1.6293996560844023e-16
julia> λ=S[2,2]
-0.5032087003825461 + 1.1969823800738464im
julia> minimum(svdvals(compute_Mder(nep,λ)))
1.1073470346550144e-15
julia> λ=S[3,3]
1.2699713558173726 + 5.342786996459857e-16im
julia> minimum(svdvals(compute_Mder(nep,λ)))
5.905315846211231e-16
julia> broyden(nep,logger=2,check_error_every=1);  # Prints out a lot more convergence info
```
In order to extract eigenpairs you can use the following:
```julia-repl
julia> (D,X)=get_deflated_eigpairs(S,V,size(nep,1));
julia> for i=1:3; @show norm(compute_Mlincomb(nep,D[i],X[:,i])); end
norm(compute_Mlincomb(nep, D[i], X[:, i])) = 8.459878994614521e-13
norm(compute_Mlincomb(nep, D[i], X[:, i])) = 1.2102336671048442e-13
norm(compute_Mlincomb(nep, D[i], X[:, i])) = 2.1012363973403225e-16
```
# References

* Jarlebring, Broyden’s method for nonlinear eigenproblems, SIAM J. Sci. Comput., 41:A989–A1012, 2019, https://arxiv.org/pdf/1802.07322

"""
broyden(nep::NEP;params...)=broyden(ComplexF64,nep,:eye;params...)
broyden(nep::NEP,approxnep;params...)=broyden(ComplexF64,nep,approxnep;params...)
broyden(T,nep::NEP;params...)=broyden(T,nep,:eye;params...)
function broyden(::Type{TT},nep::NEP,approxnep::Union{NEP,Symbol,Matrix};σ::Number=0,
                 pmax::Integer=3,
                 c::Vector=ones(TT,size(nep,1)),
                 maxit::Integer=1000,addconj=false,
                 check_error_every::Integer=10,
                 print_error_every::Integer=1,
                 threshold::Real=0.2,
                 tol::Real=1e-12,
                 errmeasure::Function=broyden_default_errmeasure,
                 add_nans::Bool=false,
                 include_restart_timing::Bool=true,
                 eigmethod::Symbol=:eig,
                 logger =0,
                 recompute_U = false,
                 inner_logger = 0
                 ) where {TT<:Number}

    @parse_logger_param!(logger)
    @parse_logger_param!(inner_logger)

    time0=time_ns();
    n=size(nep,1);
    if (pmax>size(nep,1))
        @warn "Too many eigenvalues requested. Reducing"
        pmax=size(nep,1);
    end
    σ=TT(σ);

    # Step 1. Compute M0 and T0
    if (isa(approxnep,Matrix))
        M1=approxnep;
    elseif (approxnep == :eye)
        M1=Matrix{TT}(I,n,n)
    else
        M1=compute_Mder(approxnep,σ);
    end

    T1=inv(M1);

    X=zeros(TT,n,0);
    S=zeros(TT,0,0);

    k=1;

    all_errhist=[]; sumiter=1;
    all_timehist=[];
    all_iterhist=[];
    sumiter=1;
    UU = Matrix{TT}(I, n, pmax+1) # For storage
    U1=view(UU,1:n,1:0);
    while (k<=pmax)

        # Step 5
        ## Complete recomputation of U0
        p_U1=size(U1,2);
        U1=view(UU,1:n,1:k-1);
        if recompute_U
            i_start=1;
        else
            i_start=p_U1+1; # In theory they are already satisfied
        end
        for i=i_start:k-1
            ei=zeros(TT,size(S,1)); ei[i]=1;
            f = (σ * Matrix(1.0I, k-1, k-1) - S) \ ei
            vv=X*f;
            U1[:,i]=compute_Mlincomb(nep,σ,vv);
        end

        # Step 6
        MM::Matrix{TT}=[M1 U1; X' zeros(TT,k-1,k-1)];
        push_info!(logger,"running eigval comp for deflation",continues=true);

        local d,V;
        if (eigmethod==:eig)
            d,V = eigen(MM)
        elseif (eigmethod==:eigs)
            d,V = eigs(MM,which=:SM)
        elseif (eigmethod==:invpow)
            d,V = eigs_invpow(MM,maxit=4000,sigma=0)
        else
            error("Unknown eig method",eignmethod)
        end

        push_info!(logger,"");

        x=V[:,argmin(abs.(d))];

        if (TT <: Real)
            x=real(x);
        end

        # Orthogonalize
        v0=x[1:n];
        u0=x[n+1:end];
        h=X'*v0;
        v0=v0-X*h;
        u0 = u0 + (σ * Matrix{TT}(I, k-1, k-1) - S) * h
        CH=[X';c'];

        # Normalize it
        u0=u0/(c'*v0);
        v0=v0/(c'*v0);

        if (!include_restart_timing)
            time0=time_ns();
        end

        # Step 7
        d=sqrt(eps(real(TT)));
        push_info!(logger,"Computing initial matrix",continues=true);
        f1a=(compute_Mlincomb(nep,σ+d,v0)-compute_Mlincomb(nep,σ-d,v0))/2d;
        f1b = -U1 * ((σ * Matrix{TT}(I, k-1, k-1) - S) \ u0)
        push_info!(logger,".");
        f1=f1a+f1b;
        W1=[U1 f1];

        push_info!(logger,"Starting broyden n=$n")
        T=copy(T1);

        # Main call to broyden_T
        (λm,vm,um,Tm,Wm,iter,errhist,timehist)=
        broyden_T(TT,nep,
                  v1=v0,u1=u0,λ1=σ,
                  CH=CH,T1=T1,W1=W1,
                  S=S,X=X,
                  maxit=maxit,
                  check_error_every=check_error_every,
                  print_error_every=print_error_every,
                  threshold=threshold,
                  tol=tol,
                  errmeasure=errmeasure,
                  time0=time0,
                  logger=inner_logger)

        if size(all_iterhist,1) > 0
            iterhist=(1:size(errhist,1)) .+ all_iterhist[end]
        else
            iterhist=(1:size(errhist,1))
        end

        # Deflation book keeping

        if (add_nans && size(all_iterhist,1)>1)
            all_errhist=[all_errhist;NaN]
            all_timehist=[all_timehist;NaN]
            all_iterhist=[all_iterhist;NaN]
        end
        all_errhist=[all_errhist;errhist];
        all_timehist=[all_timehist;timehist];
        all_iterhist=[all_iterhist;iterhist];
        sumiter=sumiter+iter;
        um=um/norm(vm[1:n])
        vm=vm/norm(vm[1:n]) # Normalize

        push_info!(logger,"Found an eigval $k:$λm");

        X=[X vm]
        S=[S um;zeros(1,k-1) λm]

        # Add conjugate?
        if (abs(imag(λm))>tol*10 && addconj)

            v1 = conj(vm + X[:,1:k-1] * ((λm * Matrix{TT}(I, k-1, k-1) - S[1:k-1,1:k-1]) \ um))
            λ1=conj(λm);

            rnorm=norm(compute_Mlincomb(nep,λ1,v1))
            push_info!(logger,"Adding conjugate $k")
            if (rnorm>tol*10)
                @warn "Trying to add a conjugate pair which does not have a very small residual."
            end

            h=X'*v1;
            v1t=v1-X*h;
            beta=norm(v1t);
            X=[X v1t/beta];
            k=k+1;
            S1=zeros(TT,k,k);
            S1[1:(k-1),1:(k-1)]=S;
            S1[k,k]=λ1;
            R = Matrix{TT}(I, k, k)
            R[1:k-1,end]=h; R[k,k]=beta;
            S=(R*S1)/R;
            #println("opnorm(XX-I)=",opnorm(X'*X-Matrix(1.0I, size(X,2), size(X,2))))
            #println("opnorm(MM)=",opnorm(compute_MM(nep,S,X)));
            #        X=[X v1]
            #      S=[S conj(v1[(n0+1):(n0+p)]);zeros(1,p) conj(λ1)]
            #dnep=NEPBroydenDeflated(nep,S,X);
        end
        k=k+1;
    end
    push_info!(logger,"Iterations:$sumiter")
    return S,X,T1,all_errhist,all_timehist,all_iterhist;

end

function eigs_invpow(MM;maxit=10,sigma=0)
    AA = factorize(MM - sigma * sparse(1.0I, size(MM,1), size(MM,1)))
    z=ones(size(AA,1));
    for k=1:maxit
        z=AA\z;
        z=z/opnorm(z);
    end
    lambda=z'*MM*z;
    z=reshape(z,size(AA,1),1);
    return ([lambda],z);
end


text:
module DashCoreComponents
    function __init__()
        @warn "The `DashCoreComponents` package is deprecated. All DashCoreComponents functions have been moved to the Dash package. Please switch to `using Dash` instead."
    end
end

max_stars_repo_name: plotly/dash-core-components

max_issues_repo_name: plotly/dash-core-components

max_forks_repo_name: plotly/dash-core-components


text:
module BlockCiphers

include("Rand.jl")

import Base

include("gen/libcrypto_common.jl")
include("gen/aes.h.jl")

immutable AES
    raw_key :: Array{Uint8}
    enc_key :: Array{Uint8}
    de_key :: Array{Uint8}

    function AES(key :: Array{Uint8})
        bits = length(key) * 8
        @assert (bits == 128 || bits == 192 || bits == 256)

        enc_key = zeros(Uint8, 1000)
        AES_set_encrypt_key(key, length(key) * 8, enc_key)

        de_key = zeros(Uint8, 1000)
        AES_set_decrypt_key(key, length(key) * 8, de_key)

        new(key, enc_key, de_key)
    end
end

generate_key(::Type{AES}, size) = AES(Rand.create_block_cipher_key(uint64(size / 8)))

function block_size(k :: AES)
    16
end

function encrypt!(key :: AES, data, out)
    AES_encrypt(data, out, key.enc_key)
    out
end
encrypt!(key, data) = encrypt!(key, data, data)
encrypt(key, data) = encrypt!(key, data, zeros(Uint8, block_size(key)))

function decrypt!(key :: AES, data, out)
    AES_decrypt(data, out, key.de_key)
    out
end
decrypt!(key, data) = decrypt!(key, data, data)
decrypt(key, data) = decrypt!(key, data, zeros(Uint8, block_size(key)))

include("gen/des.h.jl")

immutable DES
    raw_key :: Array{Uint8}
    lib_key :: Array{Uint8}

    function DES(key :: Array{Uint8})
        bits = length(key) * 8
        @assert bits == 64 # but 8 of the bits are for parity

        lib_key = zeros(Uint8, 200)
        DES_set_key_unchecked(key, lib_key)

        new(key, lib_key)
    end
end

function generate_key(::Type{DES}, size)
    key = zeros(Uint8, 8)
    DES_random_key(key)
    DES(key)
end

block_size(k :: DES) = 8

function encrypt!(key :: DES, data, out)
    DES_ecb_encrypt(data, out, key.lib_key, 1)
    out
end

function decrypt!(key :: DES, data, out)
    DES_ecb_encrypt(data, out, key.lib_key, 0)
    out
end

immutable DES3
    k1 :: DES
    k2 :: DES
    k3 :: DES

    function DES3(key :: Array{Uint8})
        bits = length(key) * 8
        @assert bits == 64 * 3 # but 8 of the bits are for parity

        new(DES(key[1:8]), DES(key[9:16]), DES(key[17:24]))
    end
end

function generate_key(::Type{DES3}, size)
    final_key = zeros(Uint8, 24)

    key = zeros(Uint8, 8)
    DES_random_key(key)
    final_key[1:8] = key

    DES_random_key(key)
    final_key[9:16] = key

    DES_random_key(key)
    final_key[17:24] = key

    DES3(final_key)
end

block_size(k :: DES3) = 8

function encrypt!(key :: DES3, data, out)
    encrypt!(key.k1, data, out)
    encrypt!(key.k2, out, out)
    encrypt!(key.k3, out, out)
    out
end

function decrypt!(key :: DES3, data, out)
    decrypt!(key.k3, data, out)
    decrypt!(key.k2, out, out)
    decrypt!(key.k1, out, out)
    out
end

immutable DES3ede
    k1 :: DES
    k2 :: DES
    k3 :: DES

    function DES3ede(key :: Array{Uint8})
        bits = length(key) * 8
        @assert bits == 64 * 3 # but 8 of the bits are for parity

        new(DES(key[1:8]), DES(key[9:16]), DES(key[17:24]))
    end
end

function generate_key(::Type{DES3ede}, size)
    final_key = zeros(Uint8, 24)

    key = zeros(Uint8, 8)
    DES_random_key(key)
    final_key[1:8] = key

    DES_random_key(key)
    final_key[9:16] = key

    DES_random_key(key)
    final_key[17:24] = key

    DES3ede(final_key)
end

block_size(k :: DES3ede) = 8

function encrypt!(key :: DES3ede, data, out)
    encrypt!(key.k1, data, out)
    decrypt!(key.k2, out, out)
    encrypt!(key.k3, out, out)
    out
end

function decrypt!(key :: DES3ede, data, out)
    decrypt!(key.k3, data, out)
    encrypt!(key.k2, out, out)
    decrypt!(key.k1, out, out)
    out
end

export block_size, encrypt, encrypt!, decrypt, decrypt!,
    generate_key,
    AES, DES, DES3, DES3ede

end # module BlockCiphers


text:
#####
##### diagnostics
#####

"""
$(TYPEDEF)

# Fields

$(FIELDS)
"""
struct UnivariateQuantileBoundaries{T <: AbstractVector}
    "Univariate (interior, equispaced) quantile boundaries."
    boundaries::T
end

"""
$(SIGNATURES)

Quantiles ``1/K, …, (K-1)/K`` of real numbers `xs`, as a `UnivariateQuantileBoundaries`
object.
"""
function quantile_boundaries(xs, K)
    UnivariateQuantileBoundaries(quantile(xs, (1:(K - 1)) ./ K))
end

"""
$(TYPEDEF)

Summary statistics from univariate bin counts.

# Fields

$(FIELDS)
"""
struct UnivariateBinCounts
    "Number of samples in chain."
    N::Int
    "Effective sample size correction factor."
    τ::Float64
    "Bin counts."
    bin_counts::Vector{Int}
end

"""
$(SIGNATURES)

Calculate univariate bin counts of `xs` (a vector of real numbers), using the given quantile
boundaries. Effective sample size is saved.
"""
function bin_counts(uqb::UnivariateQuantileBoundaries, xs)
    @unpack boundaries = uqb
    bin_counts = zeros(1 + length(boundaries))
    for x in xs
        bin_counts[searchsortedfirst(boundaries, x)] += 1
    end
    τ = first(ess_factor_estimate(xs))
    UnivariateBinCounts(length(xs), τ, bin_counts)
end

const ESS_CORRECTION_DOC = """
When `ess_correction = true` (the default), the standard deviation is calculated using the
the effective sample size.
"""

"""
$(SIGNATURES)

Normal approximation for bin count distributions. Return a `NamedTuple` of `μ` (the mean)
and `σ` (the standard deviation).

$(ESS_CORRECTION_DOC)
"""
function _normal_approximation(ubc::UnivariateBinCounts; ess_correction::Bool = true)
    @unpack N, τ, bin_counts = ubc
    π = 1 / length(bin_counts)
    μ = N * π
    σ = √(N / (ess_correction ? τ : one(τ)) * π * (1 - π))
    (μ = μ, σ = σ)
end

"""
$(SIGNATURES)

Calculate two-sided p-values for bin counts.

Uses a normal approxiation with continuity correction.

$(ESS_CORRECTION_DOC)
"""
function two_sided_pvalues(ubc::UnivariateBinCounts; ess_correction::Bool = true)
    @unpack μ, σ = _normal_approximation(ubc; ess_correction = ess_correction)
    @unpack bin_counts = ubc
    # use a normal approximation for p-values
    qs = normcdf.((bin_counts .+ 0.5 .- μ) ./ σ)
    map(q -> (q > 0.5 ? 1 - q : q) * 2, qs)
end

####
#### ASCII printing
####

"""
$(SIGNATURES)

Helper function for making sure everything fits into the plot area nicely.
"""
function _inflate_extrema(left, right, padding)
    @argcheck left ≤ right
    @argcheck padding ≥ 0
    Δ = right - left
    (left - Δ * padding, right + Δ * padding)
end

"""
$(SIGNATURES)

Print an ASCII plot to `io` with statistics from the given univariate bin counts. When the
first argument is a `String`, return the output as a string instead of printing.

# Keyword arguments

- `canvas_width`: width of the canvas for the plot
- `p_colname`: column name for p-values (the actual value printed is `-log10(p)`)
- `bin_colname`: column name for bin indices
- `count_colname`: column name for counts
- `α`: two markers (with `(` and `)` are placed at the quantiles `α` and `1-α`
- `padding`: enlargement factor for canvas

$(ESS_CORRECTION_DOC)
"""
function print_ascii_plot(io::IO, ubc::UnivariateBinCounts; canvas_width = 80,
                          p_colname = "-log10(p)", bin_colname = "bin", α = 0.05,
                          count_colname = "count", padding = 0.05, ess_correction = true)
    @unpack μ, σ = _normal_approximation(ubc)
    @unpack N, bin_counts = ubc
    @argcheck 0 < α < 0.5
    @argcheck canvas_width ≥ 10
    z = norminvcdf(α / 2)
    N1, N2 = μ .+ σ .* (z, -z)  # boundary lines
    Nl, Nr = _inflate_extrema(extrema(vcat(bin_counts, [N1, N2]))..., padding)
    # linear map from coordinates to canvas
    b = (canvas_width - 1) / (Nr - Nl)
    a = 0.5 - b * Nl
    f(n) = clamp(round(Int, a + n * b), 1, canvas_width)
    # draw
    K = length(bin_counts)
    canvas = fill(' ', K, canvas_width) # empty canvas
    canvas[:, f(N1)] .= '('
    canvas[:, f(N2)] .= ')'
    canvas[:, f(μ)] .= '|'
    for (i, n) in enumerate(bin_counts)
        canvas[i, f(n)] = '#'
    end
    ps = two_sided_pvalues(ubc; ess_correction = ess_correction)
    # print everything
    p_pad = max(5, length(p_colname))
    b_pad = max(length(string(K)), length(bin_colname))
    c_pad = max(length(string(maximum(bin_counts))), length(count_colname))
    canvas_label = " counts with boundaries at p-value = $(α) "
    canvas_label = lpad(canvas_label * '-'^((canvas_width - length(canvas_label) - 2) ÷ 2),
                        canvas_width - 2, '-')
    println(io, rpad(p_colname, p_pad, ' '), ' ', rpad(bin_colname, b_pad, ' '), ' ',
            rpad(count_colname, c_pad, ' '), ' ', " (", canvas_label, ")")
    for i in 1:K
        p_string = @sprintf "%.1f" -log10(ps[i])
        if length(p_string) > p_pad # should never happen
            p_string = "LARGE"
        end
        print(io, lpad(p_string, p_pad, ' '), ' ', lpad(string(i), b_pad, ' '), ' ',
              lpad(string(bin_counts[i]), c_pad), ' ')
        for c in canvas[i, :]
            print(io, c)
        end
        println(io)
    end
    nothing
end

function print_ascii_plot(::Type{String}, ubc::UnivariateBinCounts; kwargs...)
    io = IOBuffer()
    print_ascii_plot(io, ubc; kwargs...)
    String(take!(io))
end

Base.show(io::IO, ubc::UnivariateBinCounts) = print_ascii_plot(io, ubc)


text:
### Interferometers ###
"""
Supertype to any concrete interferomter type such as [`UserDefinedInterferometer`](@ref),
[`RandHaar`](@ref), [`Fourier`](@ref),...
"""
abstract type Interferometer end

"""
    UserDefinedInterferometer(U::Matrix)

Creates an instance of [`Interferometer`](@ref) from a given unitary matrix `U`.

    Fields:
        - m::Int
        - U::Matrix
"""
struct UserDefinedInterferometer <: Interferometer

    #actively checks unitarity, inefficient if outputing many matrices that are known to be unitary
    m::Int
    U::Matrix
    UserDefinedInterferometer(U) = is_unitary(U) ? new(size(U,1), U) : error("input matrix is non unitary")
end

"""
    RandHaar(m::Int)

Creates an instance of [`Interferometer`](@ref) from a Haar distributed unitary matrix of dimension `m`.

    Fields:
        - m::Int
        - U::Matrix
"""
struct RandHaar <: Interferometer
    m::Int
    U::Matrix{ComplexF64}
    RandHaar(m) = new(m,rand_haar(m))
end

"""
    Fourier(m::Int)

Creates a Fourier [`Interferometer`](@ref) of dimension `m`.

    Fields:
        - m::Int
        - U::Matrix
"""
struct Fourier <: Interferometer
    m::Int
    U::Matrix{ComplexF64}
    Fourier(m::Int) = new(m,fourier_matrix(m))
end

"""
    Hadamard(m::Int)

Creates a Hadamard [`Interferometer`](@ref) of dimension `m`.

    Fields:
        - m::Int
        - U::Matrix
"""
struct Hadamard <: Interferometer
    m::Int
    U::Matrix
    Hadamard(m::Int) = new(m,hadamard_matrix(m))
end

"""
    BeamSplitter(transmission_amplitude::Float64)

Creates a beam-splitter with tunable transmissivity.

    Fields:
        - transmission_amplitude::Float64
        - U::Matrix{ComplexF64}
        - m::Int
"""
struct BeamSplitter <: Interferometer
    transmission_amplitude::Float64
    U::Matrix
    m::Int
    BeamSplitter(transmission_amplitude) = new(transmission_amplitude, beam_splitter(transmission_amplitude),2)
end

"""
    Rotation(angle::Float64)

Creates a Rotation matrix with tunable angle.

    Fields:
        - angle::Float64
        - U::Matrix{ComplexF64}
        - m::Int
"""
struct Rotation <: Interferometer
    angle::Float64
    U::Matrix
    m::Int
    Rotation(angle) = new(angle, rotation_matrix(angle),2)
end

"""
    PhaseShift(shifted_modes::Array, param_::Array)

Creates a phase-shifter that is applied on the modes precised by shifted_modes with phase shifts given in param_.

    Fields:
        - shifted_modes::Array
        - param_::Array
        - m::Int
        - U::Matrix{ComplexF64}
"""
struct PhaseShift <: Interferometer
    shifted_modes::Array
    param_::Array
    m::Int
    U::Matrix
    PhaseShift(shifted_modes, param_) = new(shifted_modes, param_, length(shifted_modes), phase_shift(shifted_modes, param_))
end

# struct Circuit <: Interferometer
#
#     circuit_elements::Vector{Vector{Interferometer}}
#     layers::Vector
#     U::Matrix{ComplexF64}
#
#     Circuit(circuit_elements::Vector{Vector{Interferometer}}, layers::Vector) = new(circuit_elements::Vector{Vector{Interferometer}}, layers::Vector, build_circuit(circuit_elements,layers))
# end

Base.show(io::IO, interf::Interferometer) = print(io, "Interferometer :\n\n", "Type : ", typeof(interf), "\n", "m : ", interf.m)

Base.show(io::IO, interf::UserDefinedInterferometer) = begin
     print(io, "Interferometer :\n\n", "Type : ", typeof(interf), "\n", "m : ", interf.m, "\nUnitary : \n")
     pretty_table(io, interf.U)
end


text:
module Javis

using Animations
using Cairo: CairoImageSurface, image
using ColorTypes: ARGB32
using FFMPEG
using Gtk
using GtkReactive
using Images
using LaTeXStrings
using LightXML
import Luxor
import Luxor: Point, @layer
using ProgressMeter
using Random
using VideoIO

const FRAMES_SYMBOL = [:same]

abstract type Transition end
abstract type InternalTransition end

abstract type AbstractAction end

include("structs/Video.jl")
include("structs/Easing.jl")
include("structs/Rel.jl")
include("structs/Frames.jl")

"""
    Transformation

Defines a transformation which can be returned by an action to be accessible later.
See the `circ` function inside the [`javis`](@ref) as an example.

It can be accessed by another [`Action`])(@ref) using the symbol notation
like `:red_ball` in the example.

# Fields
- `p::Point`: the translation part of the transformation
- `angle::Float64`: the angle component of the transformation (in radians)
"""
mutable struct Transformation
    p::Point
    angle::Float64
end

include("structs/SubAction.jl")
include("structs/ActionSetting.jl")
include("structs/Action.jl")
include("structs/Transitions.jl")

"""
    Line

A type to define a line by two points. Can be used i.e. in [`projection`](@ref)
We mean the mathematic definition of a continuous line and not a segment of a line.

# Fields
- `p1::Point`: start point
- `p2::Point`: second point to define the line
"""
struct Line
    p1::Point
    p2::Point
end

"""
    Base.:*(m::Array{Float64,2}, transformation::Transformation)

Convert the transformation to a matrix and multiplies m*trans_matrix.
Return a new Transformation
"""
function Base.:*(m::Array{Float64,2}, transformation::Transformation)
    θ = transformation.angle
    p = transformation.p
    trans_matrix = [
        cos(θ) -sin(θ) p.x
        sin(θ) cos(θ) p.y
        0 0 1
    ]
    res = m * trans_matrix
    return Transformation(Point(gettranslation(res)...), getrotation(res))
end

include("util.jl")
include("luxor_overrides.jl")
include("backgrounds.jl")
include("svg2luxor.jl")
include("morphs.jl")
include("subaction_animations.jl")
include("javis_viewer.jl")
include("latex.jl")
include("transition2transformation.jl")
include("symbol_values.jl")

"""
    projection(p::Point, l::Line)

Return the projection of a point to a line.
"""
function projection(p::Point, l::Line)
    # move line to origin and describe it as a vector
    o = l.p1
    v = l.p2 - o
    # point also moved to origin
    x = p - o

    # scalar product <x,v>/<v,v>
    c = (x.x * v.x + x.y * v.y) / (v.x^2 + v.y^2)
    return c * v + o
end

"""
    javis(
        video::Video,
        actions::Vector{AbstractAction};
        framerate=30,
        pathname="",
        tempdirectory="",
        liveview=false
    )

Similar to `animate` in Luxor with a slightly different structure.
Instead of using actions and a video instead of scenes in a movie.

# Arguments
- `video::Video`: The video which defines the dimensions of the output
- `actions::Vector{Action}`: All actions that are performed

# Keywords
- `framerate::Int`: The frame rate of the video
- `pathname::String`: The path for the rendered gif or mp4 (i.e `output.gif` or `output.mp4`)
- `tempdirectory::String`: The folder where each frame is stored
    Defaults to a temporary directory when not set
- `liveview::Bool`: Causes a live image viewer to appear to assist with animation development

# Example
```
function ground(args...)
    background("white")
    sethue("black")
end

function circ(p=O, color="black")
    sethue(color)
    circle(p, 25, :fill)
    return Transformation(p, 0.0)
end

from = Point(-200, -200)
to = Point(-20, -130)
p1 = Point(0,-100)
p2 = Point(0,-50)
from_rot = 0.0
to_rot = 2π

demo = Video(500, 500)
javis(demo, [
    Action(1:100, ground),
    Action(1:100, :red_ball, (args...)->circ(p1, "red"), Rotation(from_rot, to_rot)),
    Action(1:100, (args...)->circ(p2, "blue"), Rotation(to_rot, from_rot, :red_ball))
], tempdirectory="images", pathname="rotating.gif")
```

This structure makes it possible to refer to positions of previous actions
i.e :red_ball is an id for the position or the red ball which can be used in the
rotation of the next ball.
"""
function javis(
    video::Video,
    actions::Vector{AA};
    framerate = 30,
    pathname = "javis_$(randstring(7)).gif",
    liveview = false,
    tempdirectory = "",
) where {AA<:AbstractAction}
    compute_frames!(actions)

    for action in actions
        compute_frames!(action.subactions; last_frames = get_frames(action))
    end

    # get all frames
    frames = Int[]
    for action in actions
        append!(frames, collect(get_frames(action)))
    end
    frames = unique(frames)

    # create internal transition objects
    for action in actions
        create_internal_transitions!(action)
        for subaction in action.subactions
            create_internal_transitions!(subaction)
        end
    end

    # create defs object
    for action in actions
        if action.id !== nothing
            video.defs[action.id] = Transformation(O, 0.0)
        end
    end

    if isempty(CURRENT_ACTION)
        push!(CURRENT_ACTION, actions[1])
    else
        CURRENT_ACTION[1] = actions[1]
    end

    if liveview == true
        _javis_viewer(video, length(frames), actions)
        return "Live preview started."
    end

    path, ext = "", ""
    if !isempty(pathname)
        path, ext = splitext(pathname)
    end
    render_mp4 = ext == ".mp4"
    codec_props = [:priv_data => ("crf" => "22", "preset" => "medium")]
    if render_mp4
        video_io = Base.open("temp.stream", "w")
    end
    video_encoder = nothing
    # if we render a gif and the user hasn't set a tempdirectory
    if !render_mp4 && isempty(tempdirectory)
        tempdirectory = mktempdir()
    end

    filecounter = 1
    @showprogress 1 "Rendering frames..." for frame in frames
        frame_image = convert.(RGB, get_javis_frame(video, actions, frame))
        if !isempty(tempdirectory)
            Images.save("$(tempdirectory)/$(lpad(filecounter, 10, "0")).png", frame_image)
        end
        if render_mp4
            if frame == first(frames)
                video_encoder = prepareencoder(
                    frame_image,
                    framerate = framerate,
                    AVCodecContextProperties = codec_props,
                )
            end
            appendencode!(video_encoder, video_io, frame_image, filecounter)
        end
        filecounter += 1
    end

    isempty(pathname) && return
    if ext == ".gif"
        # generate a colorpalette first so ffmpeg does not have to guess it
        ffmpeg_exe(`-loglevel panic -i $(tempdirectory)/%10d.png -vf
                    "palettegen=stats_mode=diff" -y "$(tempdirectory)/palette.bmp"`)
        # then apply the palette to get better results
        ffmpeg_exe(`-loglevel panic -framerate $framerate -i $(tempdirectory)/%10d.png -i
                    "$(tempdirectory)/palette.bmp" -lavfi
                    "paletteuse=dither=sierra2_4a" -y $pathname`)
    elseif ext == ".mp4"
        finishencode!(video_encoder, video_io)
        close(video_io)
        mux("temp.stream", pathname, framerate; silent = true)
    else
        @error "Currently, only gif and mp4 creation is supported. Not a $ext."
    end
    return pathname
end

"""
    get_javis_frame(video, actions, frame)

Get a frame from an animation given a video object, its actions, and frame.

# Arguments
- `video::Video`: The video which defines the dimensions of the output
- `actions::Vector{Action}`: All actions that are performed
- `frame::Int`: Specific frame to be returned

# Returns
- `Array{ARGB32, 2}` - request frame as a matrix
"""
function get_javis_frame(video, actions, frame)
    background_settings = ActionSetting()
    Drawing(video.width, video.height, :image)
    origin()
    origin_matrix = cairotojuliamatrix(getmatrix())
    # this frame needs doing, see if each of the scenes defines it
    for action in actions
        # if action is not in global layer this sets the background_settings
        # from the parent background action
        update_action_settings!(action, background_settings)
        CURRENT_ACTION[1] = action
        if frame in get_frames(action)
            # check if the action should be part of the global layer (i.e BackgroundAction)
            # or in its own layer (default)
            in_global_layer = get(action.opts, :in_global_layer, false)::Bool
            if !in_global_layer
                @layer begin
                    perform_action(action, video, frame, origin_matrix)
                end
            else
                perform_action(action, video, frame, origin_matrix)
                # update origin_matrix as it's inside the global layer
                origin_matrix = cairotojuliamatrix(getmatrix())
            end
        end
        # if action is in global layer this changes the background settings
        update_background_settings!(background_settings, action)
    end
    img = image_as_matrix()
    finish()
    return img
end

"""
    perform_action(action, video, frame, origin_matrix)

Is called inside the `javis` and does everything handled for an `AbstractAction`.
It is a 4-step process:
- compute the transformation for this action (translation, rotation, scale)
- do the transformation
- call the action function
- save the result of the action if wanted inside `video.defs`
"""
function perform_action(action, video, frame, origin_matrix)
    # first compute and perform the global transformations of this action
    compute_transition!(action, video, frame)
    perform_transformation(action)

    # relative frame number for subactions
    rel_frame = frame - first(get_frames(action)) + 1
    # call currently active subactions and their transformations
    for subaction in action.subactions
        if rel_frame in get_frames(subaction)
            subaction.func(video, action, subaction, rel_frame)
            compute_transition!(subaction, video, rel_frame)
            perform_transformation(subaction)
        elseif rel_frame > last(get_frames(subaction))
            # call the subaction on the last frame i.e. disappeared things stay disappeared
            subaction.func(video, action, subaction, last(get_frames(subaction)))
            # have the transformation from the last active frame
            compute_transition!(subaction, video, last(get_frames(subaction)))
            perform_transformation(subaction)
        end
    end

    # set the defaults for the frame like setline() and setopacity()
    # which can depend on the subactions
    set_action_defaults!(action)

    # if the scale would be 0.0 `show_action` is set to false => don't show the action
    # (it wasn't actually scaled to 0 because it would break Cairo :D)
    cs = get_current_setting()
    !cs.show_action && return

    res = action.func(video, action, frame)
    if action.id !== nothing
        current_global_matrix = cairotojuliamatrix(getmatrix())
        # obtain current matrix without the initial matrix part
        current_matrix = inv(origin_matrix) * current_global_matrix

        # if a transformation let's save the global coordinates
        if res isa Point
            vec = current_matrix * [res.x, res.y, 1.0]
            video.defs[action.id] = Point(vec[1], vec[2])
        elseif res isa Transformation
            trans = current_matrix * res
            video.defs[action.id] = trans
        else # just save the result such that it can be used as one wishes
            video.defs[action.id] = res
        end
    end
end

"""
    set_action_defaults!(action)

Set the default action values
- line_width and calls `Luxor.setline`.
- opacity and calls `Luxor.opacity`.
- scale and calls `Luxor.scale`.
"""
function set_action_defaults!(action)
    cs = action.current_setting
    current_line_width = cs.line_width * cs.mul_line_width
    Luxor.setline(current_line_width)
    current_opacity = cs.opacity * cs.mul_opacity
    Luxor.setopacity(current_opacity)

    desired_scale = cs.desired_scale .* cs.mul_scale
    scaleto(desired_scale...)
end

const LUXOR_DONT_EXPORT = [
    :boundingbox,
    :Boxmaptile,
    :Sequence,
    :setline,
    :setopacity,
    :fontsize,
    :get_fontsize,
    :scale,
    :text,
]

# Export each function from Luxor
for func in names(Luxor; imported = true)
    if !(func in LUXOR_DONT_EXPORT)
        eval(Meta.parse("import Luxor." * string(func)))
        eval(Expr(:export, func))
    end
end

export javis, latex
export Video, Action, BackgroundAction, SubAction, Rel
export Line, Translation, Rotation, Transformation, Scaling
export val, pos, ang, get_value, get_position, get_angle
export projection, morph
export appear, disappear, rotate_around, follow_path
export rev
export scaleto

# custom override of luxor extensions
export setline, setopacity, fontsize, get_fontsize, scale, text

end


text:
# Julia wrapper for header: cpl_progress.h
# Automatically generated using Clang.jl

"""
    GDALDummyProgress(double,
                      const char *,
                      void *) -> int

Stub progress function.
"""
function gdaldummyprogress(arg1, arg2, arg3)
    aftercare(ccall((:GDALDummyProgress, libgdal), Cint, (Cdouble, Cstring, Ptr{Cvoid}), arg1, arg2, arg3))
end

"""
    GDALTermProgress(double dfComplete,
                     const char * pszMessage,
                     void * pProgressArg) -> int

Simple progress report to terminal.

### Parameters
* **dfComplete**: completion ratio from 0.0 to 1.0.
* **pszMessage**: optional message.
* **pProgressArg**: ignored callback data argument.

### Returns
Always returns TRUE indicating the process should continue.
"""
function gdaltermprogress(arg1, arg2, arg3)
    aftercare(ccall((:GDALTermProgress, libgdal), Cint, (Cdouble, Cstring, Ptr{Cvoid}), arg1, arg2, arg3))
end

"""
    GDALScaledProgress(double dfComplete,
                       const char * pszMessage,
                       void * pData) -> int

Scaled progress transformer.
"""
function gdalscaledprogress(arg1, arg2, arg3)
    aftercare(ccall((:GDALScaledProgress, libgdal), Cint, (Cdouble, Cstring, Ptr{Cvoid}), arg1, arg2, arg3))
end

"""
    GDALCreateScaledProgress(double dfMin,
                             double dfMax,
                             GDALProgressFunc pfnProgress,
                             void * pData) -> void *

Create scaled progress transformer.

### Parameters
* **dfMin**: the value to which 0.0 in the sub operation is mapped.
* **dfMax**: the value to which 1.0 is the sub operation is mapped.
* **pfnProgress**: the overall progress function.
* **pData**: the overall progress function callback data.

### Returns
pointer to pass as pProgressArg to sub functions. Should be freed with GDALDestroyScaledProgress().
"""
function gdalcreatescaledprogress(arg1, arg2, arg3, arg4)
    aftercare(ccall((:GDALCreateScaledProgress, libgdal), Ptr{Cvoid}, (Cdouble, Cdouble, GDALProgressFunc, Ptr{Cvoid}), arg1, arg2, arg3, arg4))
end

"""
    GDALDestroyScaledProgress(void * pData) -> void

Cleanup scaled progress handle.

### Parameters
* **pData**: scaled progress handle returned by GDALCreateScaledProgress().
"""
function gdaldestroyscaledprogress(arg1)
    aftercare(ccall((:GDALDestroyScaledProgress, libgdal), Cvoid, (Ptr{Cvoid},), arg1))
end


text:
@testset "setup" begin

    @testset "simple random example" begin
        data_file = "$(@__DIR__)/../example/dummy.csv"
        number_observations = countlines(open(data_file))

        experiment = Dict{Symbol, Any}(
            :hash => 0,
            :data_file => data_file,
            :output_file => "OneClassActiveLearning.jl/data/output/scenarioA/data_qs_model_id.tmp",
            :model => Dict(:type => :(SVDD.RandomOCClassifier),
                           :param => Dict{Symbol, Any}(),
                           :init_strategy => SVDD.FixedParameterInitialization(GaussianKernel(2), 0.5)),
            :query_strategy => Dict(:type => :(OneClassActiveLearning.QueryStrategies.RandomQs), :param => Dict{Symbol, Any}()),
            :split_strategy => OneClassActiveLearning.DataSplits(trues(number_observations) , OneClassActiveLearning.FullSplitStrat()),
            :oracle => :PoolOracle,
            :param => Dict(:num_al_iterations => 5,
                           :solver => TEST_SOLVER,
                           :initial_pools => fill(:U, number_observations),
                           :adjust_K => true))
        expected_experiment = deepcopy(experiment)

        res = OneClassActiveLearning.active_learn(experiment)

        @test res.status[:exit_code] == :success
        @test length(res.al_history, :query_history) == 5
        @test length(res.al_history, :time_qs) == 5
        @test all(values(res.al_history, :time_qs) .> 0.0)

        @test length(res.al_history, :time_fit) == 6
        @test all(values(res.al_history, :time_fit) .> 0.0)

        @test length(res.al_history, :query_history) == 5
        @test !isempty(res.worker_info)

        @test length(res.al_history, :cm) == 6
        @test OneClassActiveLearning.cohens_kappa(last(res.al_history[:cm])[2]) ≈ last(res.al_history[:cohens_kappa])[2]

        @test res.experiment[:param][:initial_pools] == expected_experiment[:param][:initial_pools]
    end

    @testset "update pools" begin
        pools = [:U, :U, :Lin]
        labels = [:inlier, :outlier, :inlier]
        OneClassActiveLearning.update_pools!(pools, 1, :inlier)
        @test pools == [:Lin, :U, :Lin]
        OneClassActiveLearning.update_pools!(pools, 2, :outlier)
        @test pools == [:Lin, :Lout, :Lin]
    end

    @testset "get_query_object" begin
        qs = OneClassActiveLearning.TestQs()
        @testset "a" begin
            data = rand(2, 6)
            pools = fill(:U, 6)
            pools[5] = :Lin
            indices = [1, 3, 5, 7, 9, 11]
            history = [7]
            pool_map = MLLabelUtils.labelmap(pools)
            @test scores = OneClassActiveLearning.qs_score(qs, data, MLLabelUtils.labelmap(pools)) == collect(1:6)
            @test_throws ArgumentError OneClassActiveLearning.get_query_object(qs, data, fill(:Lin, 5), indices, history)
            @test OneClassActiveLearning.get_query_object(qs, data, pools, indices, history) == 11
        end

        @testset "b" begin
            data = rand(2, 5)
            pools = fill(:U, 5)
            pools[5] = :Lin
            indices = [1, 2, 4, 7, 9]
            history = [7]
            pool_map = MLLabelUtils.labelmap(pools)
            @test scores = OneClassActiveLearning.qs_score(qs, data, MLLabelUtils.labelmap(pools)) == collect(1:5)
            @test_throws ArgumentError OneClassActiveLearning.get_query_object(qs, data, fill(:Lin, 5), indices, history)
            @test OneClassActiveLearning.get_query_object(qs, data, pools, indices, history) == 4
        end
    end
end


text:
using Mill, JSON, BSON, Flux, JsonGrinder, Test, HierarchicalUtils
using JsonGrinder: suggestextractor, schema, sample_synthetic, make_representative_sample
using JsonGrinder: DictEntry, Entry, MultiEntry, ArrayEntry
using Mill: reflectinmodel

@testset "Basic behavior testing" begin
	j1 = JSON.parse("""{"a": 4, "b": {"a":[1,2,3],"b": 1},"c": { "a": {"a":[1,2,3],"b":[4,5,6]}}}""",inttype=Float64)
	j2 = JSON.parse("""{"a": 4, "c": { "a": {"a":[2,3],"b":[5,6]}}}""")
	j3 = JSON.parse("""{"a": 4, "b": {"a":[1,2,3],"b": 1}}""")
	j4 = JSON.parse("""{"a": 4, "b": {}}""")
	j5 = JSON.parse("""{"b": {}}""")
	j6 = JSON.parse("""{}""")

	sch = JsonGrinder.schema([j1,j2,j3,j4,j5,j6])

	@test sch[:a].counts == Dict(4 => 4)
	@test sch[:a].updated == 4
	@test sch[:b].updated == 4
	@test sch[:b][:a].updated == 2
	@test sch[:b][:a].l == Dict(3 => 2)
	@test sch[:b][:a].items.counts == Dict(1 => 2, 2 => 2, 3 => 2)
	@test sch[:b][:a].items.updated == 6
	@test sch[:b][:b].counts == Dict(1 => 2)
	@test sch[:b][:b].updated == 2
	@test sch[:c].updated == 2
	@test sch[:c][:a].updated == 2
	@test sch[:c][:a][:a].updated == 2
	@test sch[:c][:a][:a].l == Dict(2 => 1, 3 => 1)
	@test sch[:c][:a][:a].items.counts == Dict(1 => 1, 2 => 2, 3 => 2)
	@test sch[:c][:a][:a].items.updated == 5
	@test sch[:c][:a][:b].updated == 2
	@test sch[:c][:a][:b].l == Dict(2 => 1, 3 => 1)
	@test sch[:c][:a][:b].items.counts == Dict(4 => 1, 5 => 2, 6 => 2)
	@test sch[:c][:a][:b].items.updated == 5

	@test keys(sch[:a]) == [4]
end

@testset "Empty string vs missing key testing" begin
	j1 = JSON.parse("""{"a": "b", "b": ""}""")
	j2 = JSON.parse("""{"a": "a", "b": "c"}""")
	sch1 = JsonGrinder.schema([j1,j2])
	@test sch1[:a].updated == 2
	@test sch1[:b].updated == 2
	@test sch1.updated == 2

	j1 = JSON.parse("""{"a": "b"}""")
	j2 = JSON.parse("""{"a": "a", "b": "c"}""")
	sch2 = JsonGrinder.schema([j1,j2])
	@test sch2[:a].updated == 2
	@test sch2[:b].updated == 1
	@test sch2.updated == 2

	ext1 = suggestextractor(sch1)
	ext2 = suggestextractor(sch2)

	m1 = reflectinmodel(sch1, ext1)
	m2 = reflectinmodel(sch2, ext2)

	@test m1[:b].m isa Dense
	@test m1[:b].m.weight isa Matrix
	@test m2[:b].m isa PostImputingDense
	@test m2[:b].m.weight isa PostImputingMatrix
	@test buf_printtree(m1) != buf_printtree(m2)
end

@testset "Irregular schema" begin
	j1 = JSON.parse("""{"a": 4}""")
	j2 = JSON.parse("""{"a": { "a": "hello", "b":[5,6]}}""")
	j3 = JSON.parse("""{"a": [1, 2, 3 , 4]}""")
	# todo: add tests for array and empty dict, and which extractors it generates
	# basically test sth. like this
	# j1 = JSON.parse("""{"a": 4, "b": [{"a":[1,2,3],"b": 1}],"c": { "a": {"a":[1,2,3],"b":[4,5,6]}}}""",inttype=Float64)
	# j2 = JSON.parse("""{"a": 4, "c": {"a": {"a":[2,3],"b":[5,6]}}}""")
	# j3 = JSON.parse("""{"a": 4, "b": [{"a":[1,2,3],"b": 1}]}""")
	# j4 = JSON.parse("""{"a": 4, "b": [{}]}""")
	# j5 = JSON.parse("""{"b": {}}""")
	# j6 = JSON.parse("""{}""")

	sch = JsonGrinder.DictEntry()
	JsonGrinder.update!(sch, j1)
	@test typeof(sch[:a]) <: Entry
	JsonGrinder.update!(sch, j2)
	JsonGrinder.update!(sch, j3)
	@test typeof(sch[:a]) <: MultiEntry
	@test typeof(sch[:a][1]) <: Entry
	@test typeof(sch[:a][2]) <: DictEntry
	@test typeof(sch[:a][3]) <: ArrayEntry

	jsons = [j1, j2, j3]
	types = [Entry, DictEntry, ArrayEntry]
	for ii in [[1,2,3], [3,2,1], [2,3,1],[2,1,3] , [1,3,2]]
		sch = JsonGrinder.DictEntry()
		JsonGrinder.update!(sch, jsons[ii[1]])
		@test typeof(sch[:a]) <: types[ii[1]]
		JsonGrinder.update!(sch, jsons[ii[2]])
		JsonGrinder.update!(sch, jsons[ii[3]])
		@test typeof(sch[:a]) <: MultiEntry
		@test typeof(sch[:a][1]) <: types[ii[1]]
		@test typeof(sch[:a][2]) <: types[ii[2]]
		@test typeof(sch[:a][3]) <: types[ii[3]]
	end

	jsons = [Dict("a" => [j["a"]]) for j in jsons]
	for ii in [[1,2,3], [3,2,1], [2,3,1],[2,1,3] , [1,3,2]]
		sch = JsonGrinder.DictEntry()
		JsonGrinder.update!(sch, jsons[ii[1]])
		@test typeof(sch[:a]) <: ArrayEntry
		@test typeof(sch[:a].items) <: types[ii[1]]
		JsonGrinder.update!(sch, jsons[ii[2]])
		JsonGrinder.update!(sch, jsons[ii[3]])
		@test typeof(sch[:a].items)  <: MultiEntry
		@test typeof(sch[:a].items[1]) <: types[ii[1]]
		@test typeof(sch[:a].items[2]) <: types[ii[2]]
		@test typeof(sch[:a].items[3]) <: types[ii[3]]
	end

	j1 = JSON.parse("""{"a": 4}""")
	j2 = JSON.parse("""{"a": "hello"}""")
	sch = JsonGrinder.DictEntry()
	JsonGrinder.update!(sch, j1)
	@test typeof(sch[:a]) <: Entry
	JsonGrinder.update!(sch, j2)

end

@testset "Empty arrays" begin
	j1 = JSON.parse("""{"a": []}""")
	j2 = JSON.parse("""{"a": [{"a":1},{"b":2}]}""")
	j3 = JSON.parse("""{"a": [{"a":1,"b":3},{"b":2,"a" : 1}]}""")
	j4 = JSON.parse("""{"a": [{"a":2,"b":3}]}""")

	sch1 = JsonGrinder.schema([j1])
	sch2 = JsonGrinder.schema([j1,j2,j3])
	sch3 = JsonGrinder.schema([j2,j3,j1])

	@test sch1.updated == 1
	@test keys(sch1) == Set([:a])
	ext1 = suggestextractor(sch1)
	@test isnothing(ext1)

	@test sch2.updated == sch3.updated
	@test sch2[:a].l == sch3[:a].l
	@test sch2[:a].updated == sch3[:a].updated
	@test sch2[:a].items[:a].updated == sch3[:a].items[:a].updated
	@test sch2[:a].items[:a].counts == sch3[:a].items[:a].counts

	ext2 = suggestextractor(sch2)
	ext3 = suggestextractor(sch3)
	@test ext2 == ext3
end

@testset "More empty arrays" begin
	j1 = JSON.parse("""{"a": [{"a":1},{"b":2}], "b":[]}""")
	j2 = JSON.parse("""{"a": [{"a":1,"b":3},{"b":2,"a":1}], "b":[]}""")

	sch1 = JsonGrinder.schema([j1,j2])
	ext1 = suggestextractor(sch1)
	m = reflectinmodel(sch1, ext1)
	# todo: add tests that b is not there
	@test sch1.updated == 2
	@test sch1[:a].updated == 2
	@test :b ∈ keys(sch1)
	@test :b ∉ keys(ext1)
	@test :b ∉ keys(m)
end

@testset "Empty Arrays with multientry" begin
	j1 = JSON.parse("""{"a": [{"a":1},{"b":2}], "b":[]}""")
	j2 = JSON.parse("""{"a": [{"a":1,"b":3},{"b":2,"a":1}], "b":{}}""")
	j3 = JSON.parse("""{"a": [{"a":1,"b":3},{"b":2,"a":1}], "b":1}""")
	j4 = JSON.parse("""{"a": [{"a":1,"b":3},{"b":2,"a":1}], "b":"string"}""")

	sch1 = JsonGrinder.schema([j1,j2,j3,j4])
	ext1 = suggestextractor(sch1)
	m = reflectinmodel(sch1, ext1)
	# todo: add tests that b is not there
	@test [1,2,3,4] == keys(sch1[:b])
	@test (:e1, :e2) == keys(ext1[:b])
	@test (:e1, :e2) == keys(m[:b])
end

@testset "Schema merging" begin
	j1 = JSON.parse("""{"a": [{"a":1},{"b":2}]}""")
	j2 = JSON.parse("""{"a": [{"a":1,"b":3},{"b":2,"a" : 1}]}""")
	j3 = JSON.parse("""{"a": [{"a":2,"b":3}]}""")
	j4 = JSON.parse("""{"a": []}""")
	j5 = JSON.parse("""{}""")
	j6 = JSON.parse("""{"a": [{"a":1,"b":3},{"b":2,"a" : 1}], "b": 1}""")

	# todo: test how newentry works with array with multiple elements
	sch1 = JsonGrinder.schema([j1,j2,j3])
	sch2 = JsonGrinder.schema([j4,j5,j6])

	sch = JsonGrinder.schema([j1,j2,j3,j4,j5,j6])
	sch_merged = merge(sch1, sch2)

	@test sch.updated == sch_merged.updated
	@test sch[:a].updated == sch_merged[:a].updated
	@test sch[:a].l == sch_merged[:a].l
	@test sch[:a].items[:a].updated == sch_merged[:a].items[:a].updated
	@test sch[:a].items[:a].counts == sch_merged[:a].items[:a].counts
	@test sch[:a].items[:b].updated == sch_merged[:a].items[:b].updated
	@test sch[:a].items[:b].counts == sch_merged[:a].items[:b].counts
	@test sch[:b].updated == sch_merged[:b].updated
	@test sch[:b].counts == sch_merged[:b].counts
end

@testset "Schema merging with numbers" begin
	j1 = JSON.parse("""{"a": 1}""")
	j2 = JSON.parse("""{"a": 2}""")
	j3 = JSON.parse("""{"a": 3.3}""")
	j4 = JSON.parse("""{"a": 4.4}""")
	j5 = JSON.parse("""{"a": "5"}""")
	j6 = JSON.parse("""{"a": "6"}""")
	j7 = JSON.parse("""{"a": "7.7"}""")
	j8 = JSON.parse("""{"a": "8.8"}""")

	# todo: test how newentry works with array with multiple elements
	sch12 = JsonGrinder.schema([j1,j2])
	sch34 = JsonGrinder.schema([j3,j4])
	sch56 = JsonGrinder.schema([j5,j6])
	sch78 = JsonGrinder.schema([j7,j8])

	sch1234 = JsonGrinder.schema([j1,j2,j3,j4])
	sch5678 = JsonGrinder.schema([j5,j6,j7,j8])
	sch1256 = JsonGrinder.schema([j1,j2,j5,j6])
	sch3478 = JsonGrinder.schema([j3,j4,j7,j8])

	sch = JsonGrinder.schema([j1,j2,j3,j4,j5,j6,j7,j8])
	sch_merged1234 = merge(sch12, sch34)
	sch_merged5678 = merge(sch56, sch78)
	sch_merged1256 = merge(sch12, sch56)
	sch_merged3478 = merge(sch34, sch78)
	sch_merged = merge(sch_merged1234, sch_merged5678)
	@test sch1234 == sch_merged1234
	@test sch1234 != sch_merged5678
	@test sch5678 == sch_merged5678
	@test sch1256 == sch_merged1256
	@test sch3478 == sch_merged3478
	@test sch == sch_merged
end

@testset "Bson and symbol keys testing" begin
	function bson_buffer(data)
		b = IOBuffer()
		BSON.bson(b, data)
		seek(b, 0)
	end

	b1 = bson_buffer(Dict(:a=>4,:b=>Dict(:a=>[1,2,3], :b=>1), :c=>Dict(:a=>Dict(:a=>[1,2,3],:b=>[4,5,6]))))
	b2 = bson_buffer(Dict(:a=>4,:c=>Dict(:a=>Dict(:a=>[2,3],:b=>[5,6]))))
	b3 = bson_buffer(Dict(:a=>4,:b=>Dict(:a=>[1,2,3],:b=>1)))
	b4 = bson_buffer(Dict(:a=>4,:b=>Dict()))
	b5 = bson_buffer(Dict(:b=>Dict()))
	b6 = bson_buffer(Dict())
	bs = [BSON.BSONDict(BSON.load(b)) for b in [b1,b2,b3,b4,b5,b6]]
	sch = JsonGrinder.schema(bs)

	@test sch[:a].counts == Dict(4 => 4)
	@test sch[:a].updated == 4
	@test sch[:b].updated == 4
	@test sch[:b][:a].updated == 2
	@test sch[:b][:a].l == Dict(3 => 2)
	@test sch[:b][:a].items.counts == Dict(1 => 2, 2 => 2, 3 => 2)
	@test sch[:b][:a].items.updated == 6
	@test sch[:b][:b].counts == Dict(1 => 2)
	@test sch[:b][:b].updated == 2
	@test sch[:c].updated == 2
	@test sch[:c][:a].updated == 2
	@test sch[:c][:a][:a].updated == 2
	@test sch[:c][:a][:a].l == Dict(2 => 1, 3 => 1)
	@test sch[:c][:a][:a].items.counts == Dict(1 => 1, 2 => 2, 3 => 2)
	@test sch[:c][:a][:a].items.updated == 5
	@test sch[:c][:a][:b].updated == 2
	@test sch[:c][:a][:b].l == Dict(2 => 1, 3 => 1)
	@test sch[:c][:a][:b].items.counts == Dict(4 => 1, 5 => 2, 6 => 2)
	@test sch[:c][:a][:b].items.updated == 5
end

@testset "Equals and hash test" begin
	j1 = JSON.parse("""{"a": []}""")
	j2 = JSON.parse("""{"a": [{"a":1},{"b":2}]}""")
	j3 = JSON.parse("""{"a": [{"a":1,"b":3},{"b":2,"a" : 1}]}""")
	j4 = JSON.parse("""{"a": [{"a":2,"b":3}]}""")

	sch0 = JsonGrinder.DictEntry()
	sch1 = JsonGrinder.schema([j1, j2, j3, j4])
	sch2 = JsonGrinder.schema([j1, j2, j3, j4])
	sch3 = JsonGrinder.schema([j1, j2, j3])

	@test hash(sch1) === hash(sch2)
	@test hash(sch1) !== hash(sch3)
	@test hash(sch1) !== hash(sch0)
	@test hash(sch3) !== hash(sch0)

	@test sch1 == sch2
	@test sch1 != sch3
	@test sch1 != sch0
	@test sch3 != sch0
end

@testset "Schema merging with max keys" begin
	j1 = JSON.parse("""{"a": [{"a":1},{"b":2}]}""")
	j2 = JSON.parse("""{"a": [{"a":1,"b":3},{"b":2,"a":1}]}""")
	j3 = JSON.parse("""{"a": [{"a":2,"b":3}]}""")
	j4 = JSON.parse("""{"a": []}""")
	j5 = JSON.parse("""{}""")
	j6 = JSON.parse("""{"a": [{"a":1,"b":3},{"b":2,"a":1}], "b":1}""")
	j7 = JSON.parse("""{"a": [{"a":4,"b":5},{"b":6,"a":7}], "b":2}""")
	j8 = JSON.parse("""{"a": [{"a":9,"b":10},{"b":11,"a":12}], "b":2}""")
	j9 = JSON.parse("""{"a": [{"a":4,"b":3},{"b":2,"a":2}], "b":2}""")
	j10 = JSON.parse("""{"a": [{"a":11,"b":12},{"b":13,"a":14}], "b":2}""")
	j11 = JSON.parse("""{"a": [{"a":7,"b":5},{"b":6,"a":6}], "b":2}""")

	prev_keys = JsonGrinder.max_keys()
	JsonGrinder.updatemaxkeys!(6)
	# todo: test how newentry works with array with multiple elements
	sch1 = JsonGrinder.schema([j1,j2,j3,j4,j5,j11])
	sch2 = JsonGrinder.schema([j6,j7,j8,j9,j10])

	sch = JsonGrinder.schema([j1,j2,j3,j4,j5,j6,j7,j8,j9,j10,j11])
	sch_merged = merge(sch1, sch2)

	@test sch == sch_merged
	JsonGrinder.updatemaxkeys!(prev_keys)
end

@testset "Sample synthetic and make_representative_sample" begin
	@testset "basic without missing keys" begin
		sch = DictEntry(Dict(
			:a=>ArrayEntry(
				DictEntry(Dict(
					:a=>Entry(Dict(1=>4,2=>1), 5),
					:b=>Entry(Dict(1=>1,2=>2,3=>2), 5),
					),
				5),
				Dict(0=>1,1=>1,2=>2),
			4),
			:b=>Entry(Dict(1=>2,2=>2), 4),
			),
		4)
		@test sample_synthetic(sch) == Dict(
			:a=>[
				Dict(:a=>2,:b=>2),
				Dict(:a=>2,:b=>2)
			],
			:b=>2)
		ext = suggestextractor(sch)
		@test !ext[:a].item[:a].uniontypes
		@test !ext[:a].item[:b].uniontypes

		s = make_representative_sample(sch, ext)
		@test s isa ProductNode{NamedTuple{(:a, :b),
			Tuple{BagNode{
				ProductNode{NamedTuple{(:a, :b),
					Tuple{
						ArrayNode{OneHotMatrix{UInt32, UInt32(3), Vector{UInt32}},Nothing},
						ArrayNode{OneHotMatrix{UInt32, UInt32(4), Vector{UInt32}},Nothing}}
					},Nothing},
				AlignedBags{Int64},Nothing},
			ArrayNode{OneHotMatrix{UInt32, UInt32(3), Vector{UInt32}},Nothing}}
		}, Nothing}
	end

	@testset "basic with missing keys" begin
		sch = DictEntry(Dict(
			:a=>ArrayEntry(
				DictEntry(Dict(
					:a=>Entry(Dict(1=>3,2=>1), 4),
					:b=>Entry(Dict(2=>2,3=>2), 4),
					),
				5),
				Dict(0=>1,1=>1,2=>2),
			4),
			:b=>Entry(Dict(1=>2,2=>2), 4),
			),
		4)
		@test sample_synthetic(sch) == Dict(
			:a=>[
				Dict(:a=>2,:b=>2),
				Dict(:a=>2,:b=>2),
				],
			:b=>2
			)
		ext = suggestextractor(sch)
		@test ext[:a].item[:a].uniontypes
		@test ext[:a].item[:b].uniontypes
		@test !ext[:b].uniontypes

		s = make_representative_sample(sch, ext)
		@test s isa ProductNode{NamedTuple{(:a, :b),
			Tuple{BagNode{
				ProductNode{NamedTuple{(:a, :b),
					Tuple{
						ArrayNode{MaybeHotMatrix{Union{Missing, UInt32}, UInt32, Union{Missing, Bool}},Nothing},
						ArrayNode{MaybeHotMatrix{Union{Missing, UInt32}, UInt32, Union{Missing, Bool}},Nothing}}
				},Nothing},AlignedBags{Int64},Nothing},
			ArrayNode{OneHotMatrix{UInt32, UInt32(3), Vector{UInt32}},Nothing}
		}},Nothing}
	end

	@testset "with missing keys in dict" begin
		sch = DictEntry(Dict(
			:a=>ArrayEntry(
				DictEntry(Dict(
					:a=>Entry(Dict("a"=>1,"b"=>1,"c"=>1,"d"=>1), 4),
					:b=>Entry(Dict(3=>1,2=>2), 3),
					:c=>Entry(Dict(1=>5), 5)
					),
				5),
				Dict(0=>1,1=>1,2=>2),
			4)),
		4)

		@test JsonGrinder.sample_synthetic(sch) == Dict(
			:a=>[Dict(:a=>"c",:b=>2,:c=>1), Dict(:a=>"c",:b=>2,:c=>1)]
		)

		ext = suggestextractor(sch)
		@test ext[:a].item[:a].uniontypes
		@test ext[:a].item[:b].uniontypes
		@test !ext[:a].item[:c].uniontypes
		# todo: add tests for extraction of synthetic samples and if they have correct types
		m = reflectinmodel(sch, ext)
		# now I test that all outputs are numbers. If some output was missing, it would mean model does not have imputation it should have
		@test m(ext(JSON.parse("""{"a": [{"a":"a","c":1},{"b":2,"c":1}]}"""))).data isa Matrix{Float32}
		@test m(ext(JSON.parse("""{"a": []}"""))).data isa Matrix{Float32}

		s = make_representative_sample(sch, ext)
		@test s isa ProductNode{NamedTuple{(:a,),
			Tuple{BagNode{
				ProductNode{NamedTuple{(:a, :b, :c),
					Tuple{
						ArrayNode{MaybeHotMatrix{Union{Missing, UInt32}, UInt32, Union{Missing, Bool}}, Nothing},
						ArrayNode{MaybeHotMatrix{Union{Missing, UInt32}, UInt32, Union{Missing, Bool}}, Nothing},
						ArrayNode{Flux.OneHotArray{UInt32, 0x00000002, 1, 2, Vector{UInt32}}, Nothing}
					}},
				Nothing},
			AlignedBags{Int64}, Nothing}}}, Nothing}
	end

	@testset "with missing nested dicts" begin
		sch = DictEntry(Dict(
			:a=>DictEntry(Dict(
					:a=>Entry(Dict("a"=>1,"b"=>1,"c"=>1), 3),
					:b=>Entry(Dict(3=>1), 1),
					:c=>Entry(Dict(1=>3), 3)),
				3),
			:b=>Entry(Dict(1=>4), 4)),
		4)
		@test sample_synthetic(sch) == Dict(
			:a=>Dict(:a=>"c",:b=>3,:c=>1), :b=>1
		)

		ext = suggestextractor(sch)

		@test ext[:a][:a].uniontypes
		@test ext[:a][:b].uniontypes
		@test ext[:a][:c].uniontypes
		@test !ext[:b].uniontypes

		# todo: add test for make_representative_sample
		m = reflectinmodel(sch, ext)
		# now I test that all outputs are numbers. If some output was missing, it would mean model does not have imputation it should have
		@test m(ext(JSON.parse("""{"b":1}"""))).data isa Matrix{Float32}
		@test m(ext(JSON.parse("""{"a": {"a":"c","c":1},"b":1}"""))).data isa Matrix{Float32}

		# now I test that all outputs are numbers. If some output was missing, it would mean model does not have imputation it should have
		@test m(ext(JSON.parse("""{"a": [{"a":"a","c":1},{"b":2,"c":1}], "b": 1}"""))).data isa Matrix{Float32}
		@test m(ext(JSON.parse("""{"a": [], "b": 1}"""))).data isa Matrix{Float32}
		# the key b is always present, it should not be missing
		@test_throws ErrorException m(ext(JSON.parse("""{"a": []}""")))

		s = make_representative_sample(sch, ext)
		@test s isa ProductNode{NamedTuple{(:a, :b),
			Tuple{
				ProductNode{NamedTuple{(:a, :b, :c),
					Tuple{
						ArrayNode{MaybeHotMatrix{Union{Missing, UInt32}, UInt32, Union{Missing, Bool}}, Nothing},
						ArrayNode{MaybeHotMatrix{Union{Missing, UInt32}, UInt32, Union{Missing, Bool}}, Nothing},
						ArrayNode{MaybeHotMatrix{Union{Missing, UInt32}, UInt32, Union{Missing, Bool}}, Nothing}
					}},
				Nothing},
				ArrayNode{Flux.OneHotArray{UInt32, 0x00000002, 1, 2, Vector{UInt32}}, Nothing}
			}},
		Nothing}
	end

	@testset "with numbers and numeric strings" begin
		sch = DictEntry(Dict(
			:a=>MultiEntry([
				Entry(Dict(2=>1,5=>1), 2),
				Entry(Dict("4"=>1,"3"=>1), 2)],
			4),
			:b=>MultiEntry([
				Entry(Dict(2=>1,5=>1), 2),
				Entry(Dict("4"=>1), 1)],
			3)),
		4)

		@static if VERSION >= v"1.6.0-"
			@test sample_synthetic(sch) == Dict(:a=>5,:b=>5)
		else
			@test sample_synthetic(sch) == Dict(:a=>2,:b=>2)
		end

		ext = suggestextractor(sch)
		# this is broken, all samples are full, just once as a string, once as a number, it should not be uniontype
		@test !ext[:a][1].uniontypes
		@test ext[:b][1].uniontypes

		s = ext(sample_synthetic(sch))
		# this is wrong
		@static if VERSION >= v"1.6.0-"
			@test s[:a][:e1].data ≃ [0 0 0 1 0]'
			@test s[:b][:e1].data ≃ [0 0 1 0]'
		else
			@test s[:a][:e1].data ≃ [1 0 0 0 0]'
			@test s[:b][:e1].data ≃ [1 0 0 0]'
		end

		m = reflectinmodel(sch, ext)
		@test !(m[:a][:e1].m isa PostImputingDense)
		@test m[:b][:e1].m isa PostImputingDense

		# # now I test that all outputs are numbers. If some output was missing, it would mean model does not have imputation it should have
		@test m(ext(JSON.parse("""{"a":5}"""))).data isa Matrix{Float32}
		@test m(ext(JSON.parse("""{"a":"3"}"""))).data isa Matrix{Float32}

		s = make_representative_sample(sch, ext)
		@test s isa ProductNode{NamedTuple{(:a, :b),
			Tuple{
				ProductNode{NamedTuple{(:e1,),
					Tuple{ArrayNode{OneHotMatrix{UInt32, UInt32(5), Vector{UInt32}},Nothing}}},
					Nothing},
				ProductNode{NamedTuple{(:e1,),
					Tuple{ArrayNode{MaybeHotMatrix{Union{Missing, UInt32}, UInt32, Union{Missing, Bool}},Nothing}}
					},Nothing}
				}
			},Nothing}
	end

	@testset "with irregular schema, dict and scalars mixed" begin
		sch = DictEntry(Dict(
			:a=>MultiEntry([
				Entry(Dict(2=>1,5=>1), 2),
				DictEntry(Dict(
					:a=>Entry(Dict(3=>2), 1),
					:b=>Entry(Dict(1=>2), 2)),
				2),
				],
			4)),
		4)

		# this is not representative, but all information is inside types
		@static if VERSION >= v"1.6.0-"
			@test sample_synthetic(sch) == Dict(:a=>5)
		else
			@test sample_synthetic(sch) == Dict(:a=>2)
		end

		ext = suggestextractor(sch)
		# this is broken, all samples are full, just once as a string, once as a number, it should not be uniontype
		@test ext[:a][1].uniontypes
		@test ext[:a][2][:a].uniontypes
		@test ext[:a][2][:b].uniontypes

		s = ext(sample_synthetic(sch))
		# this is wrong
		@static if VERSION >= v"1.6.0-"
			@test s[:a][:e1].data ≃ [0 1 0]'
		else
			@test s[:a][:e1].data ≃ [1 0 0]'
		end

		m = reflectinmodel(sch, ext)
		# this is wrong, it should not be postimputing
		@test m[:a][:e1].m isa PostImputingDense

		# # now I test that all outputs are numbers. If some output was missing, it would mean model does not have imputation it should have
		@test m(ext(JSON.parse("""{"a":5}"""))).data isa Matrix{Float32}
		@test m(ext(JSON.parse("""{"a":"3"}"""))).data isa Matrix{Float32}
	end
	# todo: test schema with keyasfield
end

@testset "Merge empty lists" begin
	j1 = JSON.parse("""{"a": [{"a":1},{"b":2}], "b": []}""")
	j2 = JSON.parse("""{"a": [{"a":3},{"b":4}], "b": []}""")
	j3 = JSON.parse("""{"a": [{"a":1},{"b":3}], "b": []}""")
	j4 = JSON.parse("""{"a": [{"a":2},{"b":4}], "b": [1]}""")

	sch = JsonGrinder.schema([j1, j2, j3, j4])
	sch123 = JsonGrinder.schema([j1, j2, j3])
	sch12 = JsonGrinder.schema([j1, j2])
	sch3 = JsonGrinder.schema([j3])
	sch4 = JsonGrinder.schema([j4])
	sch_merged123 = merge(sch12, sch3)
	sch_merged1234 = merge(sch12, sch3, sch4)
	@test sch == sch_merged1234
	@test sch123 == sch_merged123
end

@testset "Symmetry" begin
	j1 = JSON.parse("""{"a": [{"a":1},{"b":2}], "b": []}""")
	j2 = JSON.parse("""{"a": [{"a":3},{"b":4}], "b": [1]}""")
	j3 = JSON.parse("""{"a": [{"a":1},{"b":3}], "b": []}""")

	sch1 = JsonGrinder.schema([j1, j2, j3])
	sch2 = JsonGrinder.schema([j2, j1, j3])
	sch3 = JsonGrinder.schema([j3, j2, j1])
	sch4 = JsonGrinder.schema([j1, j3, j2])
	@test sch1 == sch2
	@test sch1 == sch3
	@test sch2 == sch3
end

@testset "Fail empty bag extractor" begin
	ex = JsonGrinder.newentry([])
	@test isnothing(suggestextractor(ex))
end

@testset "Delete in path" begin
	j1 = JSON.parse("""{"a": 4, "b": [{"a":[1,2,3],"b": 1}],"c": { "a": {"a":[1,2,3],"b":[4,5,6]}}}""",inttype=Float64)
	j2 = JSON.parse("""{"a": 4, "c": {"a": {"a":[2,3],"b":[5,6]}}}""")
	j3 = JSON.parse("""{"a": 4, "b": [{"a":[1,2,3],"b": 1}]}""")
	j4 = JSON.parse("""{"a": 4, "b": [{}]}""")
	j5 = JSON.parse("""{"b": {}}""")
	j6 = JSON.parse("""{}""")

	sch = JsonGrinder.schema([j1,j2,j3,j4,j5,j6])
	@test children(sch[:c][:a]) == [:a=>sch[:c][:a][:a], :b=>sch[:c][:a][:b]]
	delete!(sch, ".c.a", "a")
	@test children(sch[:c][:a]) == [:b=>sch[:c][:a][:b]]
	@test children(sch[:b][1].items) == [:a=>sch[:b][1].items[:a], :b=>sch[:b][1].items[:b]]
	delete!(sch, ".b.1.[]", "a")
	@test children(sch[:b][1].items) == [:b=>sch[:b][1].items[:b]]

	suggestextractor(sch)
	# todo: add tests for this is identity is used for single-keyed multi-entry
	# todo: test indentity for multiple nested single-keyed dicts
end

@testset "Extractor from schema" begin
	j1 = JSON.parse("""{"a": 4, "b": {"a":[1,2,3],"b": 1},"c": { "a": {"a":[1,2,3],"b":[4,5,6]}}}""",inttype=Float64)
	j2 = JSON.parse("""{"a": 4, "c": {"a": {"a":[2,3],"b":[5,6]}}}""")
	j3 = JSON.parse("""{"a": 4, "b": {"a":[1,2,3],"b": 1}}""")
	j4 = JSON.parse("""{"a": 4, "b": {}}""")
	j5 = JSON.parse("""{"b": {}}""")
	j6 = JSON.parse("""{}""")

	sch = JsonGrinder.schema([j1,j2,j3,j4,j5,j6])
	ext = suggestextractor(sch, testing_settings)

	@test ext[:a] isa ExtractCategorical
	@test ext[:b][:a] isa ExtractVector{Float32}
	@test ext[:b][:b] isa ExtractScalar{Float32}
	@test ext[:c][:a][:a] isa ExtractArray{ExtractScalar{Float32}}
	@test ext[:c][:a][:b] isa ExtractArray{ExtractScalar{Float32}}

	e1 = ext(j1)
	@test e1[:a].data ≈ [1, 0]
	@test e1[:b][:a].data ≈ [1, 2, 3]
	@test e1[:b][:b].data ≈ [0]
	@test e1[:c][:a][:a].data.data ≈ [0. 0.5 1.]
	@test e1[:c][:a][:b].data.data ≈ [0. 0.5 1.]
end

@testset "Mixing substrings with strings" begin
	a, b = split("a b")
	sch = JsonGrinder.schema([Dict("k" => a), Dict("k" => b), Dict("k" => "a"), Dict("k" => "b")])
	@test sch.childs[:k].counts == Dict("a"=>2, "b"=>2)
end

function with_max_keys(f::Function, a)
    orig_val = JsonGrinder.max_keys()
    JsonGrinder.updatemaxkeys!(a)
	try
    	f()
	finally
	    JsonGrinder.updatemaxkeys!(orig_val)
	end
end

@testset "Merge inplace" begin
	j1 = JSON.parse("""{"a": 1}""")
	j2 = JSON.parse("""{"a": 2}""")
	j3 = JSON.parse("""{"a": 3}""")
	j4 = JSON.parse("""{"a": 1}""")
	j5 = JSON.parse("""{"a": 2}""")
	j6 = JSON.parse("""{"a": 3}""")

	# todo: test how newentry works with array with multiple elements
	sch1 = JsonGrinder.schema([j1,j2,j3])
	sch2 = JsonGrinder.schema([j4,j5,j6])

	@test sch1[:a].counts == Dict(2=>1,3=>1,1=>1)
	@test sch1[:a].updated == 3
	@test sch2[:a].counts == Dict(2=>1,3=>1,1=>1)
	@test sch2[:a].updated == 3
	JsonGrinder.merge_inplace!(sch1[:a], sch2[:a])
	@test sch1[:a].counts == Dict(2=>2,3=>2,1=>2)
	@test sch1[:a].updated == 6

	sch1 = JsonGrinder.schema([j1,j2,j3])
	sch2 = JsonGrinder.schema([j4,j5,j6])
	with_max_keys(2) do
		@test sch1[:a].counts == Dict(2=>1,3=>1,1=>1)
		@test sch1[:a].updated == 3
		@test sch2[:a].counts == Dict(2=>1,3=>1,1=>1)
		@test sch2[:a].updated == 3
		JsonGrinder.merge_inplace!(sch1[:a], sch2[:a])
		@test sch1[:a].counts == Dict(2=>2,3=>2)
		@test sch1[:a].updated == 6
	end
end

@testset "Merging of irregular schemas" begin
	j1 = JSON.parse("""{"a": [{"a":1},{"b":2}]}""")
	j2 = JSON.parse("""{"a": [{"a":1,"b":3},{"b":2,"a":1}]}""")
	j3 = JSON.parse("""{"a": [{"a":2,"b":3}]}""")
	j4 = JSON.parse("""{"a": []}""")
	j5 = JSON.parse("""{}""")
	j6 = JSON.parse("""{"a": [{"a":1,"b":3},{"b":2,"a":1}], "b":1}""")
	j7 = JSON.parse("""{"a": [{"a":4,"b":5},{"b":6,"a":7}], "b":2}""")
	j8 = JSON.parse("""{"a": [{"a":9,"b":10},{"b":11,"a":12}], "b":2}""")
	j9 = JSON.parse("""{"a": 4, "b":2}""")
	j10 = JSON.parse("""{"a": 11, "b":2}""")
	j11 = JSON.parse("""{"a": 7, "b":2}""")

	sch = JsonGrinder.schema([j1,j2,j3,j4,j5,j6,j7,j8,j9,j10,j11])
	sch11 = JsonGrinder.schema([j1,j2,j3,j4,j5,j6])
	sch12 = JsonGrinder.schema([j7,j8,j9])
	sch13 = JsonGrinder.schema([j10,j11])
	sch21 = JsonGrinder.schema([j1,j2,j3,j4,j5,j6,j7,j8])
	sch22 = JsonGrinder.schema([j9,j10,j11])
	sch31 = JsonGrinder.schema([j1,j2,j3,j4,j5,j6,j9])
	sch32 = JsonGrinder.schema([j8,j10])
	sch33 = JsonGrinder.schema([j11,j7])

	sch1 = merge(sch11, sch12, sch13)
	sch2 = merge(sch21, sch22)
	sch3 = merge(sch31, sch32, sch33)

	@test sch == sch1
	@test sch == sch2
	@test sch == sch3

	@test hash(sch) === hash(sch1)
	@test hash(sch) === hash(sch2)
	@test hash(sch) === hash(sch3)
end

@testset "Merging strings with numbers" begin
	j1 = JSON.parse("""{"a": "4", "b": "2"}""")
	j2 = JSON.parse("""{"a": "11", "b": "2"}""")
	j3 = JSON.parse("""{"a": 7, "b": "3"}""")
	j4 = JSON.parse("""{"a": 4, "b": 3}""")
	j5 = JSON.parse("""{"a": "11", "b": 3}""")
	j6 = JSON.parse("""{"a": "7", "b": 4}""")

	sch = JsonGrinder.schema([j1,j2,j3,j4,j5,j6])
	sch11 = JsonGrinder.schema([j1,j2])
	sch12 = JsonGrinder.schema([j3,j4])
	sch13 = JsonGrinder.schema([j5,j6])
	sch21 = JsonGrinder.schema([j1,j2,j3])
	sch22 = JsonGrinder.schema([j4,j5,j6])

	sch1 = merge(sch11, sch12, sch13)
	sch2 = merge(sch21, sch22)

	@test sch == sch1
	@test sch == sch2

	@test hash(sch) === hash(sch1)
	@test hash(sch) === hash(sch2)
end

@testset "Schema merging with max keys and irregularities" begin
	j1 = JSON.parse("""{"a": [{"a":1},{"b":2}]}""")
	j2 = JSON.parse("""{"a": [{"a":1,"b":3},{"b":2,"a":1}]}""")
	j3 = JSON.parse("""{"a": [{"a":2,"b":3}]}""")
	j4 = JSON.parse("""{"a": [1, 2, 3, 4, 5, 6, 7]}""")
	j5 = JSON.parse("""{}""")
	j6 = JSON.parse("""{"a": [{"a":1,"b":3},{"b":2,"a":2}], "b":4}""")
	j7 = JSON.parse("""{"a": [{"a":4,"b":5},{"b":6,"a":7}], "b":2}""")
	j8 = JSON.parse("""{"a": [{"a":9,"b":10},{"b":11,"a":12}], "b":"1"}""")
	j9 = JSON.parse("""{"a": [{"a":4,"b":3},{"b":2,"a":1}], "b":"2"}""")
	j10 = JSON.parse("""{"a": [{"a":11,"b":12},{"b":13,"a":"14"}], "b":"3"}""")
	j11 = JSON.parse("""{"a": [{"a":7,"b":5},{"b":6,"a":"6"}], "b":"4"}""")

	prev_keys = JsonGrinder.max_keys()
	JsonGrinder.updatemaxkeys!(4)
	# todo: test how newentry works with array with multiple elements
	sch1 = JsonGrinder.schema([j1,j2,j3,j4,j5,j11])
	sch2 = JsonGrinder.schema([j6,j7,j8,j9,j10])

	sch = JsonGrinder.schema([j1,j2,j3,j4,j5,j6,j7,j8,j9,j10,j11])
	sch_merged = merge(sch1, sch2)

	@test sch == sch_merged
	JsonGrinder.updatemaxkeys!(prev_keys)
end

@testset "Schema with string shortening" begin
	j1 = JSON.parse("""{"a": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}""")
	j2 = JSON.parse("""{"a": "b"}""")
	j3 = JSON.parse("""{"a": "🅱"}""")
	j4 = JSON.parse("""{"a": "aaaaaaaaaaa"}""")
	j5 = JSON.parse("""{"a": "aaaaaaaaaa"}""")
	j6 = JSON.parse("""{"a": "aaaaaaaaa"}""")
	j7 = JSON.parse("""{"a": "$("a"^100)"}""")

	max_string_len = 10
	sha1len = 40
	shorten_suffix = 1 + 3 + 1 + sha1len

	prev_len = JsonGrinder.max_len()
	JsonGrinder.updatemaxlen!(max_string_len)
	sch = JsonGrinder.schema([j1,j2,j3,j4,j5,j6,j7])
	@test sch[:a].counts |> keys .|> length |> maximum <= max_string_len + shorten_suffix
	@test max_string_len + shorten_suffix < 100		# sanity check that we actually shortened it

	JsonGrinder.updatemaxlen!(prev_len)
	sch = JsonGrinder.schema([j1,j2,j3,j4,j5,j6,j7])
	@test sch[:a].counts |> keys .|> length |> maximum == 100
end

@testset "Schema with string shortening" begin
	j1 = JSON.parse("""{"a": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}""")
	j2 = JSON.parse("""{"a": "b"}""")
	j3 = JSON.parse("""{"a": "🅱"}""")
	j4 = JSON.parse("""{"a": "aaaaaaaaaaa"}""")
	j5 = JSON.parse("""{"a": "aaaaaaaaaa"}""")
	j6 = JSON.parse("""{"a": "aaaaaaaaa"}""")
	j7 = JSON.parse("""{"a": "$("a"^100)"}""")

	max_string_len = 10
	sha1len = 40
	shorten_suffix = 1 + 3 + 1 + sha1len

	prev_len = JsonGrinder.max_len()
	JsonGrinder.updatemaxlen!(max_string_len)
	sch = JsonGrinder.schema([j1,j2,j3,j4,j5,j6,j7])
	@test sch[:a].counts |> keys .|> length |> maximum <= max_string_len + shorten_suffix
	@test max_string_len + shorten_suffix < 100		# sanity check that we actually shortened it

	JsonGrinder.updatemaxlen!(prev_len)
	sch = JsonGrinder.schema([j1,j2,j3,j4,j5,j6,j7])
	@test sch[:a].counts |> keys .|> length |> maximum == 100
end

@testset "Schema with fails" begin
	j1 = JSON.parse("""{"a": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}""")
	j2 = Dict("a"=>JsonGrinder)

	sch = JsonGrinder.schema([j1,j2])
	@test sch[:a].updated == 1
end

@testset "Schema with raw jsons" begin
	j1 = """{"a": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}"""
	j2 = """{"a": "b"}"""
	j3 = """{"a": "🅱"}"""
	j4 = """{"a": "aaaaaaaaaaa"}"""
	j5 = """{"a": "aaaaaaaaaa"}"""
	j6 = """{"a": "aaaaaaaaa"}"""
	j7 = """{"a": "$("a"^100)"}"""

	sch = JsonGrinder.schema([j1,j2,j3,j4,j5,j6,j7])
	@test sch[:a].updated == 7
end

@testset "prune_json" begin
	j1 = JSON.parse("""{"a": 4, "b": {"a":[1,2,3],"b": 1},"c": { "a": {"a":[1,2,3],"b":[4,5,6]}}}""",inttype=Float64)
	j2 = JSON.parse("""{"a": 4, "c": { "a": {"a":[2,3],"b":[5,6]}}}""")
	j3 = JSON.parse("""{"a": 4, "b": {"a":[1,2,3],"b": 1}}""")
	j4 = JSON.parse("""{"a": 4, "b": {}}""")
	j5 = JSON.parse("""{"b": {}}""")
	j6 = JSON.parse("""{}""")

	sch = JsonGrinder.schema([j1,j2,j3,j4,j5,j6])

	@test JsonGrinder.prune_json(j1, sch) == Dict{String, Any}(
		"c" => Dict("a"=>Dict("b"=>[4.0, 5.0, 6.0],"a"=>[1.0, 2.0, 3.0])),
		"b" => Dict("b"=>1.0,"a"=>[1.0, 2.0, 3.0]),
		"a" => 4.0)

	@test JsonGrinder.prune_json(j2, sch) == Dict{String, Any}(
		"c" => Dict("a"=>Dict("b"=>[5, 6],"a"=>[2, 3])),
		"a" => 4)

	delete!(sch.childs, :b)

	@test JsonGrinder.prune_json(j1, sch) == Dict{String, Any}(
  		"c" => Dict("a"=>Dict("b"=>[4.0, 5.0, 6.0],"a"=>[1.0, 2.0, 3.0])),
  		"a" => 4.0)

	@test JsonGrinder.prune_json(j2, sch) == Dict{String, Any}(
		"c" => Dict("a"=>Dict("b"=>[5, 6],"a"=>[2, 3])),
		"a" => 4)

	delete!(sch.childs, :c)

	@test JsonGrinder.prune_json(j1, sch) == Dict{String, Any}(
  		"a" => 4.0)

	@test JsonGrinder.prune_json(j2, sch) == Dict{String, Any}(
		"a" => 4)

	j1 = JSON.parse("""{"a": 4, "b": {"a":1, "b": 1}}""")
	j2 = JSON.parse("""{"a": 4, "b": {"a":1}}""")
	sch = JsonGrinder.schema([j1,j2])

	j3 = Dict(
		"a" => 4,
		"b" => Dict("a"=>1),
		"c" => 1,
		"d" => 2,
	)

	@test JsonGrinder.prune_json(j1, sch) == j1
	@test JsonGrinder.prune_json(j2, sch) == j2
	@test JsonGrinder.prune_json(j3, sch) == Dict(
		"a"=>4,
		"b"=>Dict(
			"a" => 1
		)
	)
end

num_params(m) = m |> params .|> size .|> prod |> sum
params_empty(m) = m |> params .|> size |> isempty

@testset "suggestextractor with ints and floats numeric and stringy" begin
	j1 = JSON.parse("""{"a": "4"}""")
	j2 = JSON.parse("""{"a": "11.5"}""")
	j3 = JSON.parse("""{"a": 7}""")
	j4 = JSON.parse("""{"a": 4.5}""")

	sch1234 = JsonGrinder.schema([j1,j2,j3,j4])
	sch123 = JsonGrinder.schema([j1,j2,j3])
	sch12 = JsonGrinder.schema([j1,j2])
	sch23 = JsonGrinder.schema([j2,j3])
	sch14 = JsonGrinder.schema([j1,j4])
	sch34 = JsonGrinder.schema([j3,j4])

	ext1234 = suggestextractor(sch1234)
	ext123 = suggestextractor(sch123)
	ext12 = suggestextractor(sch12)
	ext23 = suggestextractor(sch23)
	ext34 = suggestextractor(sch34)
	ext14 = suggestextractor(sch14)

	# as expected, sometimes there is multirepresentation
	@test buf_printtree(ext12, trav=true) ==
	"""
	Dict [""]
	  └── a: Categorical d = 3 ["U"]
	"""

	@test buf_printtree(ext23, trav=true) ==
  	"""
	Dict [""]
	  └── a: MultiRepresentation ["U"]
	           └── e1: Categorical d = 3 ["k"]
	"""

  	@test buf_printtree(ext34, trav=true) ==
	"""
	Dict [""]
	  └── a: Categorical d = 3 ["U"]
	"""

  	@test buf_printtree(ext14, trav=true) ==
	"""
	Dict [""]
	  └── a: MultiRepresentation ["U"]
	           └── e1: Categorical d = 3 ["k"]
	"""

	# but that's not problem, there are identity layers, so number of parameters is same

	m12 = reflectinmodel(sch12, ext12)
	m23 = reflectinmodel(sch23, ext23)
	m34 = reflectinmodel(sch34, ext34)
	m14 = reflectinmodel(sch14, ext14)

	# testing that I have same numer of params
	@test num_params(m12) == 40
	@test num_params(m12) == num_params(m23)
	@test num_params(m12) == num_params(m34)
	@test num_params(m12) == num_params(m14)

	# now now with scalars
	ext1234 = suggestextractor(sch1234, testing_settings)
	ext123 = suggestextractor(sch123, testing_settings)
	ext12 = suggestextractor(sch12, testing_settings)
	ext23 = suggestextractor(sch23, testing_settings)
	ext34 = suggestextractor(sch34, testing_settings)
	ext14 = suggestextractor(sch14, testing_settings)

	# as expected, sometimes there is multirepresentation
	@test buf_printtree(ext12, trav=true) ==
	"""
	Dict [""]
	  └── a: Float32 ["U"]
	"""

	@test buf_printtree(ext23, trav=true) ==
  	"""
	Dict [""]
	  └── a: MultiRepresentation ["U"]
	           └── e1: Float32 ["k"]
	"""

  	@test buf_printtree(ext34, trav=true) ==
	"""
	Dict [""]
	  └── a: Float32 ["U"]
	"""

  	@test buf_printtree(ext14, trav=true) ==
	"""
	Dict [""]
	  └── a: MultiRepresentation ["U"]
	           └── e1: Float32 ["k"]
	"""

	# but that's not problem, there are identity layers, so number of parameters is same

	# by default there is only 1 scalar and indentities, so they have not params
	m12 = reflectinmodel(sch12, ext12)
	m23 = reflectinmodel(sch23, ext23)
	m34 = reflectinmodel(sch34, ext34)
	m14 = reflectinmodel(sch14, ext14)

	# testing that I have no params in all models
	@test params_empty(m12)
	@test params_empty(m23)
	@test params_empty(m34)
	@test params_empty(m14)
end

@testset "is_numeric is_floatable is_intable" begin
	j1 = JSON.parse("""{"a": "4"}""")
	j2 = JSON.parse("""{"a": "11.5"}""")
	j3 = JSON.parse("""{"a": 7}""")
	j4 = JSON.parse("""{"a": 4.5}""")

	sch1234 = JsonGrinder.schema([j1,j2,j3,j4])
	sch12 = JsonGrinder.schema([j1,j2])
	sch34 = JsonGrinder.schema([j3,j4])
	sch13 = JsonGrinder.schema([j1,j3])
	sch3 = JsonGrinder.schema([j3])
	sch1 = JsonGrinder.schema([j1])
	e = sch1234[:a]

	expected_multientry = JsonGrinder.MultiEntry([
		JsonGrinder.Entry(Dict("4"=>1,"11.5"=>1),2),
		JsonGrinder.Entry(Dict(7=>1,4.5=>1),2)
	], 4)
	@test e == expected_multientry

	e_hash = hash(e)
	@test JsonGrinder.merge_entries_with_cast(e, Int32, Real) == expected_multientry
	# checking that merge_entries_with_cast is not mutating the argument
	@test e_hash == hash(e)

	expected_merged = JsonGrinder.MultiEntry([
		JsonGrinder.Entry(Dict(7=>1,4.0=>1,11.5=>1,4.5=>1),4)
	], 4)
	@test JsonGrinder.merge_entries_with_cast(e, JsonGrinder.FloatType, Real) == expected_merged
	# checking that merge_entries_with_cast is not mutating the argument
	@test e_hash == hash(e)
	@test JsonGrinder.merge_entries_with_cast(e, Int32, Real) != expected_merged

	e1234 = JsonGrinder.merge_entries_with_cast(e, JsonGrinder.FloatType, Real)[1]

	@test sch12[:a] == JsonGrinder.Entry(Dict("4" => 1, "11.5"=> 1),2)

	@test !JsonGrinder.is_intable(e1234)
	@test !JsonGrinder.is_floatable(e1234)
	@test JsonGrinder.is_numeric_entry(e1234, Real)

	@test !JsonGrinder.is_intable(sch12[:a])
	@test JsonGrinder.is_floatable(sch12[:a])
	@test !JsonGrinder.is_numeric_entry(sch12[:a], Real)
	# todo: fix it and make some predicates with true for both e1 and sch12[:a]
	@test !JsonGrinder.is_intable(sch34[:a])
	@test !JsonGrinder.is_floatable(sch34[:a])
	@test JsonGrinder.is_numeric_entry(sch34[:a], Real)

	expected_merged = JsonGrinder.MultiEntry([
		JsonGrinder.Entry(Dict(7=>1,4=>1),2)
	], 2)
	@test JsonGrinder.merge_entries_with_cast(sch13[:a], Int32, Real) == expected_merged

	e13 = JsonGrinder.merge_entries_with_cast(sch13[:a], Int32, Real)[1]

	@test !JsonGrinder.is_intable(e13)
	@test !JsonGrinder.is_floatable(e13)
	@test JsonGrinder.is_numeric_entry(e13, Real)

	@test !JsonGrinder.is_intable(sch3[:a])
	@test !JsonGrinder.is_floatable(sch3[:a])
	@test JsonGrinder.is_numeric_entry(sch3[:a], Real)

	@test JsonGrinder.is_intable(sch1[:a])
	@test JsonGrinder.is_floatable(sch1[:a])
	@test !JsonGrinder.is_numeric_entry(sch1[:a], Real)
end