text:
module TestBroadcasting

using Test, DataFrames, PooledArrays, Random, CategoricalArrays

const ≅ = isequal

refdf = DataFrame(reshape(1.5:15.5, (3, 5)), :auto)

@testset "CartesianIndex" begin
    df = DataFrame(rand(2, 3), :auto)
    for i in axes(df, 1), j in axes(df, 2)
        @test df[i, j] == df[CartesianIndex(i, j)]
        r = rand()
        df[CartesianIndex(i, j)] = r
        @test df[i, j] == r
    end
    @test_throws BoundsError df[CartesianIndex(0, 1)]
    @test_throws BoundsError df[CartesianIndex(0, 0)]
    @test_throws BoundsError df[CartesianIndex(1, 0)]
    @test_throws BoundsError df[CartesianIndex(5, 1)]
    @test_throws BoundsError df[CartesianIndex(5, 5)]
    @test_throws BoundsError df[CartesianIndex(1, 5)]

    @test_throws BoundsError df[CartesianIndex(0, 1)] = 1
    @test_throws ArgumentError df[CartesianIndex(0, 0)] = 1
    @test_throws ArgumentError df[CartesianIndex(1, 0)] = 1
    @test_throws BoundsError df[CartesianIndex(5, 1)] = 1
    @test_throws ArgumentError df[CartesianIndex(5, 5)] = 1
    @test_throws ArgumentError df[CartesianIndex(1, 5)] = 1
end

@testset "broadcasting of AbstractDataFrame objects" begin
    for df in (copy(refdf), view(copy(refdf), :, :))
        @test identity.(df) == refdf
        @test identity.(df) !== df
        @test (x->x).(df) == refdf
        @test (x->x).(df) !== df
        @test (df .+ df) ./ 2 == refdf
        @test (df .+ df) ./ 2 !== df
        @test df .+ Matrix(df) == 2 .* df
        @test Matrix(df) .+ df == 2 .* df
        @test (Matrix(df) .+ df .== 2 .* df) == DataFrame(trues(size(df)), names(df))
        @test df .+ 1 == df .+ ones(size(df))
        @test df .+ axes(df, 1) == DataFrame(Matrix(df) .+ axes(df, 1), names(df))
        @test df .+ permutedims(axes(df, 2)) == DataFrame(Matrix(df) .+ permutedims(axes(df, 2)), names(df))
    end

    df1 = copy(refdf)
    df2 = view(copy(refdf), :, :)
    @test (df1 .+ df2) ./ 2 == refdf
    @test (df1 .- df2) == DataFrame(zeros(size(refdf)), names(refdf))
    @test (df1 .* df2) == refdf .^ 2
    @test (df1 ./ df2) == DataFrame(ones(size(refdf)), names(refdf))
end

@testset "broadcasting of AbstractDataFrame objects thrown exceptions" begin
    df = copy(refdf)
    dfv = view(df, :, 2:ncol(df))

    @test_throws DimensionMismatch df .+ dfv
    @test_throws DimensionMismatch df .+ df[2:end, :]

    @test_throws DimensionMismatch df .+ [1, 2]
    @test_throws DimensionMismatch df .+ [1 2]
    @test_throws DimensionMismatch df .+ rand(2, 2)
    @test_throws DimensionMismatch dfv .+ [1, 2]
    @test_throws DimensionMismatch dfv .+ [1 2]
    @test_throws DimensionMismatch dfv .+ rand(2, 2)

    df2 = copy(df)
    rename!(df2, [:x1, :x2, :x3, :x4, :y])
    @test_throws ArgumentError df .+ df2
    @test_throws ArgumentError df .+ 1 .+ df2
end

@testset "broadcasting data frames" begin
    df1 = DataFrame(x=1, y=2)
    df2 = DataFrame(x=[1, 11], y=[2, 12])
    @test df1 .+ df2 == DataFrame(x=[2, 12], y=[4, 14])

    df1 = DataFrame(x=1, y=2)
    df2 = DataFrame(x=[1, 11], y=[2, 12])
    x = df2.x
    y = df2.y
    df2 .+= df1
    @test df2.x === x
    @test df2.y === y
    @test df2 == DataFrame(x=[2, 12], y=[4, 14])

    df = DataFrame(x=[1, 11], y=[2, 12])
    dfv = view(df, 1:1, 1:2)
    df .-= dfv
    @test df == DataFrame(x=[0, 10], y=[0, 10])

    @test DataFrame() .+ DataFrame() == DataFrame()
    @test_throws ArgumentError DataFrame(a=1, b=1) .+ DataFrame(b=1, a=1)

    df = DataFrame(a=1, b=2)
    @test_throws ArgumentError df .= DataFrame(b=1, a=2)
    @test_throws ArgumentError df .= DataFrame(a=1, c=2)
    @test_throws ArgumentError df[!, [:a, :b]] .= DataFrame(b=1, a=2)
    @test_throws ArgumentError df[!, [:a, :b]] .= DataFrame(a=1, c=2)
end

@testset "broadcasting of AbstractDataFrame objects corner cases" begin
    df = DataFrame(c11 = categorical(["a", "b"]), c12 = categorical([missing, "b"]), c13 = categorical(["a", missing]),
                   c21 = categorical([1, 2]), c22 = categorical([missing, 2]), c23 = categorical([1, missing]),
                   p11 = PooledArray(["a", "b"]), p12 = PooledArray([missing, "b"]), p13 = PooledArray(["a", missing]),
                   p21 = PooledArray([1, 2]), p22 = PooledArray([missing, 2]), p23 = PooledArray([1, missing]),
                   b1 = [true, false], b2 = [missing, false], b3 = [true, missing],
                   f1 = [1.0, 2.0], f2 = [missing, 2.0], f3 = [1.0, missing],
                   s1 = ["a", "b"], s2 = [missing, "b"], s3 = ["a", missing])

    df2 = DataFrame(c11 = categorical(["a", "b"]), c12 = [nothing, "b"], c13 = ["a", nothing],
                    c21 = categorical([1, 2]), c22 = [nothing, 2], c23 = [1, nothing],
                    p11 = ["a", "b"], p12 = [nothing, "b"], p13 = ["a", nothing],
                    p21 = [1, 2], p22 = [nothing, 2], p23 = [1, nothing],
                    b1 = [true, false], b2 = [nothing, false], b3 = [true, nothing],
                    f1 = [1.0, 2.0], f2 = [nothing, 2.0], f3 = [1.0, nothing],
                    s1 = ["a", "b"], s2 = [nothing, "b"], s3 = ["a", nothing])

    @test df ≅ identity.(df)
    @test df ≅ (x->x).(df)
    df3 = coalesce.(df, nothing)
    @test df2 == df3
    @test eltype.(eachcol(df2)) == eltype.(eachcol(df3))
    for i in axes(df, 2)
        @test typeof(df2[!, i]) == typeof(df3[!, i])
    end
    df4 = (x -> df[1, 1]).(df)
    @test names(df4) == names(df)
    @test all(isa.(eachcol(df4), Ref(CategoricalArray)))
    @test all(eachcol(df4) .== Ref(categorical(["a", "a"])))

    df5 = DataFrame(x = Any[1, 2, 3], y = Any[1, 2.0, big(3)])
    @test identity.(df5) == df5
    @test (x->x).(df5) == df5
    @test df5 .+ 1 == DataFrame(Matrix(df5) .+ 1, names(df5))
    @test eltype.(eachcol(identity.(df5))) == [Int, BigFloat]
    @test eltype.(eachcol((x->x).(df5))) == [Int, BigFloat]
    @test eltype.(eachcol(df5 .+ 1)) == [Int, BigFloat]
end

@testset "normal data frame and data frame row in broadcasted assignment - one column" begin
    df = copy(refdf)
    df[!, 1] .+= 1
    @test df.x1 == [2.5, 3.5, 4.5]
    @test df[:, 2:end] == refdf[:, 2:end]

    dfv = @view df[1:2, 2:end]
    @test_throws ArgumentError dfv[!, 1] .+= 1

    df = copy(refdf)
    df[:, 1] .+= 1
    @test df.x1 == [2.5, 3.5, 4.5]
    @test df[:, 2:end] == refdf[:, 2:end]

    dfv = @view df[1:2, 2:end]
    dfv.x2 .+= 1
    @test dfv.x2 == [5.5, 6.5]
    @test dfv[:, 2:end] == refdf[1:2, 3:end]
    @test Matrix(df) == [2.5  5.5  7.5  10.5  13.5
                         3.5  6.5  8.5  11.5  14.5
                         4.5  6.5  9.5  12.5  15.5]

    dfr = df[1, 3:end]
    dfr[end-1:end] .= 10
    @test Vector(dfr) == [7.5, 10.0, 10.0]
    @test Matrix(df) == [2.5  5.5  7.5  10.0  10.0
                         3.5  6.5  8.5  11.5  14.5
                         4.5  6.5  9.5  12.5  15.5]

    df = copy(refdf)
    df[!, 1] .+= [1, 2, 3]
    @test df.x1 == [2.5, 4.5, 6.5]
    @test df[:, 2:end] == refdf[:, 2:end]

    df = copy(refdf)
    df.x1 .+= [1, 2, 3]
    @test df.x1 == [2.5, 4.5, 6.5]
    @test df[:, 2:end] == refdf[:, 2:end]

    dfv = @view df[1:2, 2:end]
    dfv.x2 .+= [1, 2]
    @test dfv.x2 == [5.5, 7.5]
    @test dfv[:, 2:end] == refdf[1:2, 3:end]
    @test Matrix(df) == [2.5  5.5  7.5  10.5  13.5
                         4.5  7.5  8.5  11.5  14.5
                         6.5  6.5  9.5  12.5  15.5]

    dfr = df[1, 3:end]
    dfr[end-1:end] .= [10, 11]
    @test Vector(dfr) == [7.5, 10.0, 11.0]
    @test Matrix(df) == [2.5  5.5  7.5  10.0  11.0
                         4.5  7.5  8.5  11.5  14.5
                         6.5  6.5  9.5  12.5  15.5]

    df = copy(refdf)
    df[:, 1] .+= [1, 2, 3]
    @test df.x1 == [2.5, 4.5, 6.5]
    @test df[:, 2:end] == refdf[:, 2:end]

    dfv = @view df[1:2, 2:end]
    dfv[:, 1] .+= [1, 2]
    @test dfv.x2 == [5.5, 7.5]
    @test dfv[:, 2:end] == refdf[1:2, 3:end]
    @test Matrix(df) == [2.5  5.5  7.5  10.5  13.5
                         4.5  7.5  8.5  11.5  14.5
                         6.5  6.5  9.5  12.5  15.5]

    # test a more complex broadcasting pattern
    df = copy(refdf)
    df[!, 1] .+= [0, 1, 2] .+ 1
    @test df.x1 == [2.5, 4.5, 6.5]
    @test df[:, 2:end] == refdf[:, 2:end]

    df = copy(refdf)
    df[!, "x1"] .+= [0, 1, 2] .+ 1
    @test df."x1" == [2.5, 4.5, 6.5]
    @test df[:, Not("x1")] == refdf[:, 2:end]

    dfv = @view df[1:2, 2:end]
    @test_throws ArgumentError dfv[!, 1] .+= [0, 1] .+ 1

    dfv = @view df[1:2, 2:end]
    @test_throws ArgumentError dfv[!, "x1"] .+= [0, 1] .+ 1

    df = copy(refdf)
    df.x1 .+= [0, 1, 2] .+ 1
    @test df.x1 == [2.5, 4.5, 6.5]
    @test df[:, 2:end] == refdf[:, 2:end]

    dfv = @view df[1:2, 2:end]
    dfv.x2 .+= [0, 1] .+ 1
    @test dfv.x2 == [5.5, 7.5]
    @test dfv[:, 2:end] == refdf[1:2, 3:end]
    @test Matrix(df) == [2.5  5.5  7.5  10.5  13.5
                         4.5  7.5  8.5  11.5  14.5
                         6.5  6.5  9.5  12.5  15.5]

    df = copy(refdf)
    df."x1" .+= [0, 1, 2] .+ 1
    @test df."x1" == [2.5, 4.5, 6.5]
    @test df[:, 2:end] == refdf[:, 2:end]

    dfv = @view df[1:2, 2:end]
    dfv."x2" .+= [0, 1] .+ 1
    @test dfv."x2" == [5.5, 7.5]
    @test dfv[:, 2:end] == refdf[1:2, 3:end]
    @test Matrix(df) == [2.5  5.5  7.5  10.5  13.5
                         4.5  7.5  8.5  11.5  14.5
                         6.5  6.5  9.5  12.5  15.5]

    dfr = df[1, 3:end]
    dfr[end-1:end] .= [9, 10] .+ 1
    @test Vector(dfr) == [7.5, 10.0, 11.0]
    @test Matrix(df) == [2.5  5.5  7.5  10.0  11.0
                         4.5  7.5  8.5  11.5  14.5
                         6.5  6.5  9.5  12.5  15.5]

    df = copy(refdf)
    df[:, 1] .+= [0, 1, 2] .+ 1
    @test df.x1 == [2.5, 4.5, 6.5]
    @test df[:, 2:end] == refdf[:, 2:end]

    dfv = @view df[1:2, 2:end]
    dfv[:, 1] .+= [0, 1] .+ 1
    @test dfv.x2 == [5.5, 7.5]
    @test dfv[:, 2:end] == refdf[1:2, 3:end]
    @test Matrix(df) == [2.5  5.5  7.5  10.5  13.5
                         4.5  7.5  8.5  11.5  14.5
                         6.5  6.5  9.5  12.5  15.5]

    df = copy(refdf)
    df[:, "x1"] .+= [0, 1, 2] .+ 1
    @test df."x1" == [2.5, 4.5, 6.5]
    @test df[:, 2:end] == refdf[:, 2:end]

    dfv = @view df[1:2, 2:end]
    dfv[:, "x2"] .+= [0, 1] .+ 1
    @test dfv."x2" == [5.5, 7.5]
    @test dfv[:, 2:end] == refdf[1:2, 3:end]
    @test Matrix(df) == [2.5  5.5  7.5  10.5  13.5
                         4.5  7.5  8.5  11.5  14.5
                         6.5  6.5  9.5  12.5  15.5]

    df = copy(refdf)
    dfv = @view df[1:2, 2:end]
    dfr = df[1, 3:end]
    @test_throws DimensionMismatch df[!, 1] .= rand(3, 1)
    @test_throws ArgumentError dfv[!, 1] .= rand(2, 1)
    @test_throws DimensionMismatch dfr[end-1:end] .= rand(3, 1)
    @test_throws DimensionMismatch df[:, 1] .= rand(3, 1)
    @test_throws DimensionMismatch dfv[:, 1] .= rand(2, 1)
    @test_throws DimensionMismatch df[!, 1] .= reshape(rand(3), :, 1)
    @test_throws ArgumentError dfv[!, 1] .= reshape(rand(2), :, 1)
    @test_throws DimensionMismatch dfr[end-1:end] .= reshape(rand(3), :, 1)
    @test_throws DimensionMismatch df[:, 1] .= reshape(rand(3), :, 1)
    @test_throws DimensionMismatch dfv[:, 1] .= reshape(rand(2), :, 1)

    df = copy(refdf)
    df[!, :x1] .+= 1
    @test df.x1 == [2.5, 3.5, 4.5]
    @test df[:, 2:end] == refdf[:, 2:end]

    dfv = @view df[1:2, 2:end]
    @test_throws ArgumentError dfv[!, :x2] .+= 1

    dfr = df[1, 3:end]
    dfr[[:x4, :x5]] .= 10
    @test Vector(dfr) == [7.5, 10.0, 10.0]
    @test Matrix(df) == [2.5  4.5  7.5  10.0  10.0
                         3.5  5.5  8.5  11.5  14.5
                         4.5  6.5  9.5  12.5  15.5]

    df = copy(refdf)
    df[!, "x1"] .+= 1
    @test df."x1" == [2.5, 3.5, 4.5]
    @test df[:, 2:end] == refdf[:, 2:end]

    dfv = @view df[1:2, 2:end]
    @test_throws ArgumentError dfv[!, "x2"] .+= 1

    dfr = df[1, 3:end]
    dfr[["x4", "x5"]] .= 10
    @test Vector(dfr) == [7.5, 10.0, 10.0]
    @test Matrix(df) == [2.5  4.5  7.5  10.0  10.0
                         3.5  5.5  8.5  11.5  14.5
                         4.5  6.5  9.5  12.5  15.5]

    df = copy(refdf)
    df[:, :x1] .+= 1
    @test df.x1 == [2.5, 3.5, 4.5]
    @test df[:, 2:end] == refdf[:, 2:end]

    dfv = @view df[1:2, 2:end]
    dfv[:, :x2] .+= 1
    @test dfv.x2 == [5.5, 6.5]
    @test dfv[:, 2:end] == refdf[1:2, 3:end]
    @test Matrix(df) == [2.5  5.5  7.5  10.5  13.5
                         3.5  6.5  8.5  11.5  14.5
                         4.5  6.5  9.5  12.5  15.5]

    df = copy(refdf)
    df[:, "x1"] .+= 1
    @test df."x1" == [2.5, 3.5, 4.5]
    @test df[:, 2:end] == refdf[:, 2:end]

    dfv = @view df[1:2, 2:end]
    dfv[:, "x2"] .+= 1
    @test dfv."x2" == [5.5, 6.5]
    @test dfv[:, 2:end] == refdf[1:2, 3:end]
    @test Matrix(df) == [2.5  5.5  7.5  10.5  13.5
                         3.5  6.5  8.5  11.5  14.5
                         4.5  6.5  9.5  12.5  15.5]

    df = copy(refdf)
    df[!, :x1] .+= [1, 2, 3]
    @test df.x1 == [2.5, 4.5, 6.5]
    @test df[:, 2:end] == refdf[:, 2:end]

    dfv = @view df[1:2, 2:end]
    @test_throws ArgumentError dfv[!, :x2] .+= [1, 2]

    dfr = df[1, 3:end]
    dfr[[:x4, :x5]] .= [10, 11]
    @test Vector(dfr) == [7.5, 10.0, 11.0]
    @test Matrix(df) == [2.5  4.5  7.5  10.0  11.0
                         4.5  5.5  8.5  11.5  14.5
                         6.5  6.5  9.5  12.5  15.5]

    df = copy(refdf)
    df[!, "x1"] .+= [1, 2, 3]
    @test df."x1" == [2.5, 4.5, 6.5]
    @test df[:, 2:end] == refdf[:, 2:end]

    dfv = @view df[1:2, 2:end]
    @test_throws ArgumentError dfv[!, :x2] .+= [1, 2]

    dfr = df[1, 3:end]
    dfr[["x4", "x5"]] .= [10, 11]
    @test Vector(dfr) == [7.5, 10.0, 11.0]
    @test Matrix(df) == [2.5  4.5  7.5  10.0  11.0
                         4.5  5.5  8.5  11.5  14.5
                         6.5  6.5  9.5  12.5  15.5]

    df = copy(refdf)
    df[:, :x1] .+= [1, 2, 3]
    @test df.x1 == [2.5, 4.5, 6.5]
    @test df[:, 2:end] == refdf[:, 2:end]

    dfv = @view df[1:2, 2:end]
    dfv[:, :x2] .+= [1, 2]
    @test dfv.x2 == [5.5, 7.5]
    @test dfv[:, 2:end] == refdf[1:2, 3:end]
    @test Matrix(df) == [2.5  5.5  7.5  10.5  13.5
                         4.5  7.5  8.5  11.5  14.5
                         6.5  6.5  9.5  12.5  15.5]

    df = copy(refdf)
    df[:, "x1"] .+= [1, 2, 3]
    @test df."x1" == [2.5, 4.5, 6.5]
    @test df[:, 2:end] == refdf[:, 2:end]

    dfv = @view df[1:2, 2:end]
    dfv[:, "x2"] .+= [1, 2]
    @test dfv."x2" == [5.5, 7.5]
    @test dfv[:, 2:end] == refdf[1:2, 3:end]
    @test Matrix(df) == [2.5  5.5  7.5  10.5  13.5
                         4.5  7.5  8.5  11.5  14.5
                         6.5  6.5  9.5  12.5  15.5]

    df = copy(refdf)
    dfv = @view df[1:2, 2:end]
    dfr = df[1, 3:end]
    @test_throws DimensionMismatch df[!, :x1] .= rand(3, 1)
    @test_throws ArgumentError dfv[!, :x2] .= rand(2, 1)
    @test_throws DimensionMismatch dfr[[:x4, :x5]] .= rand(3, 1)
    @test_throws DimensionMismatch df[:, :x1] .= rand(3, 1)
    @test_throws DimensionMismatch dfv[:, :x2] .= rand(2, 1)
    @test_throws DimensionMismatch df[!, 1] .= reshape(rand(3), :, 1)
    @test_throws ArgumentError dfv[!, 1] .= reshape(rand(2), :, 1)
    @test_throws DimensionMismatch dfr[end-1:end] .= reshape(rand(3), :, 1)
    @test_throws DimensionMismatch df[:, 1] .= reshape(rand(3), :, 1)
    @test_throws DimensionMismatch dfv[:, 1] .= reshape(rand(2), :, 1)
    @test_throws DimensionMismatch df[!, "x1"] .= rand(3, 1)
    @test_throws ArgumentError dfv[!, "x2"] .= rand(2, 1)
    @test_throws DimensionMismatch dfr[["x4", "x5"]] .= rand(3, 1)
    @test_throws DimensionMismatch df[:, "x1"] .= rand(3, 1)
    @test_throws DimensionMismatch dfv[:, "x2"] .= rand(2, 1)
end

@testset "normal data frame and data frame view in broadcasted assignment - two columns" begin
    df = copy(refdf)
    df[:, [1, 2]] .= Matrix(df[:, [1, 2]]) .+ 1
    @test df.x1 == [2.5, 3.5, 4.5]
    @test df.x2 == [5.5, 6.5, 7.5]
    @test df[:, 3:end] == refdf[:, 3:end]

    dfv = @view df[1:2, 3:end]
    dfv[:, [1, 2]] .= Matrix(dfv[:, [1, 2]]) .+ 1
    @test dfv.x3 == [8.5, 9.5]
    @test dfv.x4 == [11.5, 12.5]
    @test dfv[:, 3:end] == refdf[1:2, 5:end]
    @test Matrix(df) == [2.5  5.5  8.5  11.5  13.5
                         3.5  6.5  9.5  12.5  14.5
                         4.5  7.5  9.5  12.5  15.5]

    df = copy(refdf)
    df[:, [1, 2]] .= Matrix(df[:, [1, 2]]) .+ 1
    @test df.x1 == [2.5, 3.5, 4.5]
    @test df.x2 == [5.5, 6.5, 7.5]
    @test df[:, 3:end] == refdf[:, 3:end]

    dfv = @view df[1:2, 3:end]
    dfv[:, [1, 2]] .= Matrix(dfv[:, [1, 2]]) .+ 1
    @test dfv.x3 == [8.5, 9.5]
    @test dfv.x4 == [11.5, 12.5]
    @test dfv[:, 3:end] == refdf[1:2, 5:end]
    @test Matrix(df) == [2.5  5.5  8.5  11.5  13.5
                         3.5  6.5  9.5  12.5  14.5
                         4.5  7.5  9.5  12.5  15.5]

    df = copy(refdf)
    df[:, [1, 2]] .= Matrix(df[:, [1, 2]]) .+ [1 4
                                               2 5
                                               3 6]
    @test df.x1 == [2.5, 4.5, 6.5]
    @test df.x2 == [8.5, 10.5, 12.5]
    @test df[:, 3:end] == refdf[:, 3:end]

    dfv = @view df[1:2, 3:end]
    dfv[:, [1, 2]] .= Matrix(dfv[:, [1, 2]]) .+ [1 3
                                               2 4]
    @test dfv.x3 == [8.5, 10.5]
    @test dfv.x4 == [13.5, 15.5]
    @test dfv[:, 3:end] == refdf[1:2, 5:end]
    @test Matrix(df) == [2.5   8.5   8.5  13.5  13.5
                         4.5  10.5  10.5  15.5  14.5
                         6.5  12.5   9.5  12.5  15.5]

    df = copy(refdf)
    df[:, [1, 2]] .= Matrix(df[:, [1, 2]]) .+ [1 4
                                               2 5
                                               3 6]
    @test df.x1 == [2.5, 4.5, 6.5]
    @test df.x2 == [8.5, 10.5, 12.5]
    @test df[:, 3:end] == refdf[:, 3:end]

    dfv = @view df[1:2, 3:end]
    dfv[:, [1, 2]] .= Matrix(dfv[:, [1, 2]]) .+ [1 3
                                               2 4]
    @test dfv.x3 == [8.5, 10.5]
    @test dfv.x4 == [13.5, 15.5]
    @test dfv[:, 3:end] == refdf[1:2, 5:end]
    @test Matrix(df) == [2.5   8.5   8.5  13.5  13.5
                         4.5  10.5  10.5  15.5  14.5
                         6.5  12.5   9.5  12.5  15.5]

    df = copy(refdf)
    dfv = @view df[1:2, 2:end]
    @test_throws DimensionMismatch df[:, [1, 2]] .= rand(3, 10)
    @test_throws DimensionMismatch dfv[:, [1, 2]] .= rand(2, 10)
    @test_throws DimensionMismatch df[:, [1, 2]] .= rand(3, 10)
    @test_throws DimensionMismatch dfv[:, [1, 2]] .= rand(2, 10)

    df = copy(refdf)
    df[:, [:x1, :x2]] .= Matrix(df[:, [:x1, :x2]]) .+ 1
    @test df.x1 == [2.5, 3.5, 4.5]
    @test df.x2 == [5.5, 6.5, 7.5]
    @test df[:, 3:end] == refdf[:, 3:end]

    dfv = @view df[1:2, 3:end]
    dfv[:, [:x3, :x4]] .= Matrix(dfv[:, [:x3, :x4]]) .+ 1
    @test dfv.x3 == [8.5, 9.5]
    @test dfv.x4 == [11.5, 12.5]
    @test dfv[:, 3:end] == refdf[1:2, 5:end]
    @test Matrix(df) == [2.5  5.5  8.5  11.5  13.5
                         3.5  6.5  9.5  12.5  14.5
                         4.5  7.5  9.5  12.5  15.5]

    df = copy(refdf)
    df[:, ["x1", "x2"]] .= Matrix(df[:, [:x1, :x2]]) .+ 1
    @test df.x1 == [2.5, 3.5, 4.5]
    @test df.x2 == [5.5, 6.5, 7.5]
    @test df[:, 3:end] == refdf[:, 3:end]

    dfv = @view df[1:2, 3:end]
    dfv[:, ["x3", "x4"]] .= Matrix(dfv[:, [:x3, :x4]]) .+ 1
    @test dfv.x3 == [8.5, 9.5]
    @test dfv.x4 == [11.5, 12.5]
    @test dfv[:, 3:end] == refdf[1:2, 5:end]
    @test Matrix(df) == [2.5  5.5  8.5  11.5  13.5
                         3.5  6.5  9.5  12.5  14.5
                         4.5  7.5  9.5  12.5  15.5]

    df = copy(refdf)
    df[:, [:x1, :x2]] .= Matrix(df[:, [:x1, :x2]]) .+ 1
    @test df.x1 == [2.5, 3.5, 4.5]
    @test df.x2 == [5.5, 6.5, 7.5]
    @test df[:, 3:end] == refdf[:, 3:end]

    dfv = @view df[1:2, 3:end]
    dfv[:, [:x3, :x4]] .= Matrix(dfv[:, [:x3, :x4]]) .+ 1
    @test dfv.x3 == [8.5, 9.5]
    @test dfv.x4 == [11.5, 12.5]
    @test dfv[:, 3:end] == refdf[1:2, 5:end]
    @test Matrix(df) == [2.5  5.5  8.5  11.5  13.5
                         3.5  6.5  9.5  12.5  14.5
                         4.5  7.5  9.5  12.5  15.5]

    df = copy(refdf)
    df[:, ["x1", "x2"]] .= Matrix(df[:, [:x1, :x2]]) .+ 1
    @test df.x1 == [2.5, 3.5, 4.5]
    @test df.x2 == [5.5, 6.5, 7.5]
    @test df[:, 3:end] == refdf[:, 3:end]

    dfv = @view df[1:2, 3:end]
    dfv[:, ["x3", "x4"]] .= Matrix(dfv[:, [:x3, :x4]]) .+ 1
    @test dfv.x3 == [8.5, 9.5]
    @test dfv.x4 == [11.5, 12.5]
    @test dfv[:, 3:end] == refdf[1:2, 5:end]
    @test Matrix(df) == [2.5  5.5  8.5  11.5  13.5
                         3.5  6.5  9.5  12.5  14.5
                         4.5  7.5  9.5  12.5  15.5]

    df = copy(refdf)
    df[:, [:x1, :x2]] .= Matrix(df[:, [:x1, :x2]]) .+ [1 4
                                                       2 5
                                                       3 6]
    @test df.x1 == [2.5, 4.5, 6.5]
    @test df.x2 == [8.5, 10.5, 12.5]
    @test df[:, 3:end] == refdf[:, 3:end]

    dfv = @view df[1:2, 3:end]
    dfv[:, [:x3, :x4]] .= Matrix(dfv[:, [:x3, :x4]]) .+ [1 3
                                                         2 4]
    @test dfv.x3 == [8.5, 10.5]
    @test dfv.x4 == [13.5, 15.5]
    @test dfv[:, 3:end] == refdf[1:2, 5:end]
    @test Matrix(df) == [2.5   8.5   8.5  13.5  13.5
                         4.5  10.5  10.5  15.5  14.5
                         6.5  12.5   9.5  12.5  15.5]

    df = copy(refdf)
    df[:, ["x1", "x2"]] .= Matrix(df[:, ["x1", "x2"]]) .+ [1 4
                                                           2 5
                                                           3 6]
    @test df.x1 == [2.5, 4.5, 6.5]
    @test df.x2 == [8.5, 10.5, 12.5]
    @test df[:, 3:end] == refdf[:, 3:end]

    dfv = @view df[1:2, 3:end]
    dfv[:, ["x3", "x4"]] .= Matrix(dfv[:, ["x3", "x4"]]) .+ [1 3
                                                             2 4]
    @test dfv.x3 == [8.5, 10.5]
    @test dfv.x4 == [13.5, 15.5]
    @test dfv[:, 3:end] == refdf[1:2, 5:end]
    @test Matrix(df) == [2.5   8.5   8.5  13.5  13.5
                         4.5  10.5  10.5  15.5  14.5
                         6.5  12.5   9.5  12.5  15.5]

    df = copy(refdf)
    df[:, [:x1, :x2]] .= Matrix(df[:, [:x1, :x2]]) .+ [1 4
                                                       2 5
                                                       3 6]
    @test df.x1 == [2.5, 4.5, 6.5]
    @test df.x2 == [8.5, 10.5, 12.5]
    @test df[:, 3:end] == refdf[:, 3:end]

    dfv = @view df[1:2, 3:end]
    dfv[:, [:x3, :x4]] .= Matrix(dfv[:, [:x3, :x4]]) .+ [1 3
                                                         2 4]
    @test dfv.x3 == [8.5, 10.5]
    @test dfv.x4 == [13.5, 15.5]
    @test dfv[:, 3:end] == refdf[1:2, 5:end]
    @test Matrix(df) == [2.5   8.5   8.5  13.5  13.5
                         4.5  10.5  10.5  15.5  14.5
                         6.5  12.5   9.5  12.5  15.5]

    df = copy(refdf)
    df[:, ["x1", "x2"]] .= Matrix(df[:, ["x1", "x2"]]) .+ [1 4
                                                           2 5
                                                           3 6]
    @test df.x1 == [2.5, 4.5, 6.5]
    @test df.x2 == [8.5, 10.5, 12.5]
    @test df[:, 3:end] == refdf[:, 3:end]

    dfv = @view df[1:2, 3:end]
    dfv[:, ["x3", "x4"]] .= Matrix(dfv[:, ["x3", "x4"]]) .+ [1 3
                                                             2 4]
    @test dfv.x3 == [8.5, 10.5]
    @test dfv.x4 == [13.5, 15.5]
    @test dfv[:, 3:end] == refdf[1:2, 5:end]
    @test Matrix(df) == [2.5   8.5   8.5  13.5  13.5
                         4.5  10.5  10.5  15.5  14.5
                         6.5  12.5   9.5  12.5  15.5]

    df = copy(refdf)
    dfv = @view df[1:2, 2:end]
    @test_throws DimensionMismatch df[:, [:x1, :x2]] .= rand(3, 10)
    @test_throws DimensionMismatch dfv[:, [:x3, :x4]] .= rand(2, 10)
    @test_throws DimensionMismatch df[:, [:x1, :x2]] .= rand(3, 10)
    @test_throws DimensionMismatch dfv[:, [:x3, :x4]] .= rand(2, 10)
    @test_throws DimensionMismatch df[:, ["x1", "x2"]] .= rand(3, 10)
    @test_throws DimensionMismatch dfv[:, ["x3", "x4"]] .= rand(2, 10)
    @test_throws DimensionMismatch df[:, ["x1", "x2"]] .= rand(3, 10)
    @test_throws DimensionMismatch dfv[:, ["x3", "x4"]] .= rand(2, 10)

    df = copy(refdf)
    df[:, [1, 2]] .= [1 2
                      3 4
                      5 6]
    @test Matrix(df) == [1.0  2.0  7.5  10.5  13.5
                         3.0  4.0  8.5  11.5  14.5
                         5.0  6.0  9.5  12.5  15.5]

    df = copy(refdf)
    df[:, [1, 2]] .= [1, 3, 5]
    @test Matrix(df) == [1.0  1.0  7.5  10.5  13.5
                         3.0  3.0  8.5  11.5  14.5
                         5.0  5.0  9.5  12.5  15.5]

    df = copy(refdf)
    df[:, [1, 2]] .= reshape([1, 3, 5], 3, 1)
    @test Matrix(df) == [1.0  1.0  7.5  10.5  13.5
                         3.0  3.0  8.5  11.5  14.5
                         5.0  5.0  9.5  12.5  15.5]

    df = copy(refdf)
    df[:, [1, 2]] .= 1
    @test Matrix(df) == [1.0  1.0  7.5  10.5  13.5
                         1.0  1.0  8.5  11.5  14.5
                         1.0  1.0  9.5  12.5  15.5]

    df = copy(refdf)
    dfv = view(df, 2:3, 2:4)
    dfv[:, [1, 2]] .= [1 2
                       3 4]
    @test Matrix(df) == [1.5  4.5  7.5  10.5  13.5
                         2.5  1.0  2.0  11.5  14.5
                         3.5  3.0  4.0  12.5  15.5]

    df = copy(refdf)
    dfv = view(df, 2:3, 2:4)
    dfv[:, [1, 2]] .= [1, 3]
    @test Matrix(df) == [1.5  4.5  7.5  10.5  13.5
                         2.5  1.0  1.0  11.5  14.5
                         3.5  3.0  3.0  12.5  15.5]

    df = copy(refdf)
    dfv = view(df, 2:3, 2:4)
    dfv[:, [1, 2]] .= reshape([1, 3], 2, 1)
    @test Matrix(df) == [1.5  4.5  7.5  10.5  13.5
                         2.5  1.0  1.0  11.5  14.5
                         3.5  3.0  3.0  12.5  15.5]

    df = copy(refdf)
    dfv = view(df, 2:3, 2:4)
    dfv[:, [1, 2]] .= 1
    @test Matrix(df) == [1.5  4.5  7.5  10.5  13.5
                         2.5  1.0  1.0  11.5  14.5
                         3.5  1.0  1.0  12.5  15.5]
end

@testset "assignment to a whole data frame and data frame row" begin
    df = copy(refdf)
    df .= 10
    @test all(Matrix(df) .== 10)
    dfv = view(df, 1:2, 1:4)
    dfv .= 100
    @test Matrix(df) == [100.0  100.0  100.0  100.0  10.0
                        100.0  100.0  100.0  100.0  10.0
                         10.0   10.0   10.0   10.0  10.0]
    dfr = df[1, 1:2]
    dfr .= 1000
    @test Matrix(df) == [1000.0  1000.0  100.0  100.0  10.0
                         100.0   100.0  100.0  100.0  10.0
                         10.0    10.0   10.0   10.0  10.0]

    df = copy(refdf)
    df[:, :] .= 10
    @test all(Matrix(df) .== 10)
    dfv = view(df, 1:2, 1:4)
    dfv[:, :] .= 100
    @test Matrix(df) == [100.0  100.0  100.0  100.0  10.0
                         100.0  100.0  100.0  100.0  10.0
                         10.0   10.0   10.0   10.0  10.0]
    dfr = df[1, 1:2]
    dfr[:] .= 1000
    @test Matrix(df) == [1000.0  1000.0  100.0  100.0  10.0
                         100.0   100.0   100.0  100.0  10.0
                         10.0    10.0    10.0   10.0   10.0]

    df = copy(refdf)
    df[:, :] .= 10
    @test all(Matrix(df) .== 10)
    dfv = view(df, 1:2, 1:4)
    dfv[:, :] .= 100
    @test Matrix(df) == [100.0  100.0  100.0  100.0  10.0
                         100.0  100.0  100.0  100.0  10.0
                         10.0   10.0   10.0   10.0  10.0]
end

@testset "extending data frame in broadcasted assignment - one column" begin
    df = copy(refdf)
    df[!, :a] .= 1
    @test Matrix(df) == [1.5  4.5  7.5  10.5  13.5  1.0
                         2.5  5.5  8.5  11.5  14.5  1.0
                         3.5  6.5  9.5  12.5  15.5  1.0]
    @test names(df)[end] == "a"
    @test df[:, 1:end-1] == refdf
    df[!, :b] .= [1, 2, 3]
    @test Matrix(df) == [1.5  4.5  7.5  10.5  13.5  1.0 1.0
                         2.5  5.5  8.5  11.5  14.5  1.0 2.0
                         3.5  6.5  9.5  12.5  15.5  1.0 3.0]
    @test names(df)[end] == "b"
    @test df[:, 1:end-2] == refdf
    cdf = copy(df)
    @test_throws DimensionMismatch df[!, :c] .= ones(3, 1)
    @test df == cdf
    @test_throws DimensionMismatch df[!, :x] .= ones(4)
    @test df == cdf
    @test_throws ArgumentError df[!, 10] .= ones(3)
    @test df == cdf

    dfv = @view df[1:2, 2:end]
    @test_throws ArgumentError dfv[!, 10] .= ones(3)
    @test_throws ArgumentError dfv[!, :z] .= ones(3)
    @test df == cdf
    dfr = df[1, 3:end]
    @test_throws BoundsError dfr[10] .= ones(3)
    @test_throws ArgumentError dfr[:z] .= ones(3)
    @test df == cdf

    df = DataFrame()
    @test_throws DimensionMismatch df[!, :a] .= sin.(1:3)
    df[!, :b] .= sin.(1)
    df[!, :c] .= sin(1) .+ 1
    @test df == DataFrame(b=Float64[], c=Float64[])

    df = copy(refdf)
    df[!, "a"] .= 1
    @test Matrix(df) == [1.5  4.5  7.5  10.5  13.5  1.0
                         2.5  5.5  8.5  11.5  14.5  1.0
                         3.5  6.5  9.5  12.5  15.5  1.0]
    @test names(df)[end] == "a"
    @test df[:, 1:end-1] == refdf
    df[!, "b"] .= [1, 2, 3]
    @test Matrix(df) == [1.5  4.5  7.5  10.5  13.5  1.0 1.0
                         2.5  5.5  8.5  11.5  14.5  1.0 2.0
                         3.5  6.5  9.5  12.5  15.5  1.0 3.0]
    @test names(df)[end] == "b"
    @test df[:, 1:end-2] == refdf
    cdf = copy(df)
    @test_throws DimensionMismatch df[!, "c"] .= ones(3, 1)
    @test df == cdf
    @test_throws DimensionMismatch df[!, "x"] .= ones(4)
    @test df == cdf
    @test_throws ArgumentError df[!, 10] .= ones(3)
    @test df == cdf

    dfv = @view df[1:2, 2:end]
    @test_throws ArgumentError dfv[!, 10] .= ones(3)
    @test_throws ArgumentError dfv[!, "z"] .= ones(3)
    @test df == cdf
    dfr = df[1, 3:end]
    @test_throws BoundsError dfr[10] .= ones(3)
    @test_throws ArgumentError dfr["z"] .= ones(3)
    @test df == cdf

    df = DataFrame()
    @test_throws DimensionMismatch df[!, "a"] .= sin.(1:3)
    df[!, "b"] .= sin.(1)
    df[!, "c"] .= sin(1) .+ 1
    @test df == DataFrame(b=Float64[], c=Float64[])
end

@testset "empty data frame corner case" begin
    df = DataFrame()
    @test_throws ArgumentError df[!, 1] .= 1
    @test_throws ArgumentError df[!, 2] .= 1
    @test_throws ArgumentError df[!, [:a, :b]] .= [1]
    @test_throws ArgumentError df[!, [:a, :b]] .= 1
    @test_throws DimensionMismatch df[!, :a] .= [1 2]
    @test_throws DimensionMismatch df[!, :a] .= [1, 2]
    @test_throws DimensionMismatch df[!, :a] .= sin.(1) .+ [1, 2]
    @test_throws ArgumentError df[!, ["a", "b"]] .= [1]
    @test_throws ArgumentError df[!, ["a", "b"]] .= 1
    @test_throws DimensionMismatch df[!, "a"] .= [1 2]
    @test_throws DimensionMismatch df[!, "a"] .= [1, 2]
    @test_throws DimensionMismatch df[!, "a"] .= sin.(1) .+ [1, 2]

    for rhs in [1, [1], Int[], "abc", ["abc"]]
        df = DataFrame()
        df[!, :a] .= rhs
        @test size(df) == (0, 1)
        @test eltype(df[!, 1]) == (rhs isa AbstractVector ? eltype(rhs) : typeof(rhs))

        df = DataFrame()
        df[!, :a] .= length.(rhs)
        @test size(df) == (0, 1)
        @test eltype(df[!, 1]) == Int

        df = DataFrame()
        df[!, :a] .= length.(rhs) .+ 1
        @test size(df) == (0, 1)
        @test eltype(df[!, 1]) == Int

        df = DataFrame()
        @. df[!, :a] = length(rhs) + 1
        @test size(df) == (0, 1)
        @test eltype(df[!, 1]) == Int

        df = DataFrame(x=Int[])
        df[!, :a] .= rhs
        @test size(df) == (0, 2)
        @test eltype(df[!, 2]) == (rhs isa AbstractVector ? eltype(rhs) : typeof(rhs))

        df = DataFrame(x=Int[])
        df[!, :a] .= length.(rhs)
        @test size(df) == (0, 2)
        @test eltype(df[!, 2]) == Int

        df = DataFrame(x=Int[])
        df[!, :a] .= length.(rhs) .+ 1
        @test size(df) == (0, 2)
        @test eltype(df[!, 2]) == Int

        df = DataFrame(x=Int[])
        @. df[!, :a] = length(rhs) + 1
        @test size(df) == (0, 2)
        @test eltype(df[!, 2]) == Int

        df = DataFrame()
        df[!, "a"] .= rhs
        @test size(df) == (0, 1)
        @test eltype(df[!, 1]) == (rhs isa AbstractVector ? eltype(rhs) : typeof(rhs))

        df = DataFrame()
        df[!, "a"] .= length.(rhs)
        @test size(df) == (0, 1)
        @test eltype(df[!, 1]) == Int

        df = DataFrame()
        df[!, "a"] .= length.(rhs) .+ 1
        @test size(df) == (0, 1)
        @test eltype(df[!, 1]) == Int

        df = DataFrame()
        @. df[!, "a"] = length(rhs) + 1
        @test size(df) == (0, 1)
        @test eltype(df[!, 1]) == Int

        df = DataFrame(x=Int[])
        df[!, "a"] .= rhs
        @test size(df) == (0, 2)
        @test eltype(df[!, 2]) == (rhs isa AbstractVector ? eltype(rhs) : typeof(rhs))

        df = DataFrame(x=Int[])
        df[!, "a"] .= length.(rhs)
        @test size(df) == (0, 2)
        @test eltype(df[!, 2]) == Int

        df = DataFrame(x=Int[])
        df[!, "a"] .= length.(rhs) .+ 1
        @test size(df) == (0, 2)
        @test eltype(df[!, 2]) == Int

        df = DataFrame(x=Int[])
        @. df[!, "a"] = length(rhs) + 1
        @test size(df) == (0, 2)
        @test eltype(df[!, 2]) == Int
    end

    df = DataFrame()
    df .= 1
    @test df == DataFrame()
    df .= [1]
    @test df == DataFrame()
    df .= ones(1, 1)
    @test df == DataFrame()
    @test_throws DimensionMismatch df .= ones(1, 2)
    @test_throws DimensionMismatch df .= ones(1, 1, 1)

    df = DataFrame(a=[])
    df[!, :b] .= sin.(1)
    @test eltype(df.b) == Float64
    df[!, :b] .= [1]
    @test eltype(df.b) == Int
    df[!, :b] .= 'a'
    @test eltype(df.b) == Char
    @test names(df) == ["a", "b"]

    c = categorical(["a", "b", "c"])
    df = DataFrame()
    @test_throws DimensionMismatch df[!, :a] .= c

    df[!, :b] .= c[1]
    @test nrow(df) == 0
    @test df.b isa CategoricalVector{String}

    df = DataFrame(a=[])
    df[!, "b"] .= sin.(1)
    @test eltype(df."b") == Float64
    df[!, "b"] .= [1]
    @test eltype(df."b") == Int
    df[!, "b"] .= 'a'
    @test eltype(df."b") == Char
    @test names(df) == ["a", "b"]

    c = categorical(["a", "b", "c"])
    df = DataFrame()
    @test_throws DimensionMismatch df[!, "a"] .= c

    df[!, "b"] .= c[1]
    @test nrow(df) == 0
    @test df."b" isa CategoricalVector{String}
end

@testset "test categorical values" begin
    for v in [categorical([1, 2, 3]), categorical([1, 2, missing]),
              categorical([missing, 1, 2]),
              categorical(["1", "2", "3"]), categorical(["1", "2", missing]),
              categorical([missing, "1", "2"])]
        df = copy(refdf)
        df[!, :c1] .= v
        @test df.c1 ≅ v
        @test df.c1 !== v
        @test df.c1 isa CategoricalVector
        @test levels(df.c1) == levels(v)
        @test levels(df.c1) !== levels(v)
        df[!, :c2] .= v[2]
        @test df.c2 == fill(v[2], 3)
        @test df.c2 isa CategoricalVector
        @test levels(df.c2) == levels(v)
        df[!, :c3] .= (x->x).(v)
        @test df.c3 ≅ v
        @test df.c3 !== v
        @test df.c3 isa CategoricalVector
        @test levels(df.c3) == levels(v)
        @test levels(df.c3) !== levels(v)
        df[!, :c4] .= identity.(v)
        @test df.c4 ≅ v
        @test df.c4 !== v
        @test df.c4 isa CategoricalVector
        @test levels(df.c4) == levels(v)
        @test levels(df.c4) !== levels(v)
        df[!, :c5] .= (x->v[2]).(v)
        @test unique(df.c5) == [get(v[2])]
        @test df.c5 isa CategoricalVector
        @test levels(df.c5) == levels(v)
    end
end

@testset "scalar broadcasting" begin
    a = DataFrame(x = zeros(2))
    a .= 1 ./ (1 + 2)
    @test a.x == [1/3, 1/3]
    a .= 1 ./ (1 .+ 3)
    @test a.x == [1/4, 1/4]
    a .= sqrt.(1 ./ 2)
    @test a.x == [sqrt(1/2), sqrt(1/2)]
end

@testset "tuple broadcasting" begin
    X = DataFrame(zeros(2, 3), :auto)
    X .= (1, 2)
    @test X == DataFrame([1 1 1; 2 2 2], :auto)

    X = DataFrame(zeros(2, 3), :auto)
    X .= (1, 2) .+ 10 .- X
    @test X == DataFrame([11 11 11; 12 12 12], :auto)

    X = DataFrame(zeros(2, 3), :auto)
    X .+= (1, 2) .+ 10
    @test X == DataFrame([11 11 11; 12 12 12], :auto)

    df = DataFrame(rand(2, 3), :auto)
    @test floor.(Int, df ./ (1,)) == DataFrame(zeros(Int, 2, 3), :auto)
    df .= floor.(Int, df ./ (1,))
    @test df == DataFrame(zeros(2, 3), :auto)

    df = DataFrame(rand(2, 3), :auto)
    @test_throws InexactError convert.(Int, df)
    df2 = convert.(Int, floor.(df))
    @test df2 == DataFrame(zeros(Int, 2, 3), :auto)
    @test eltype.(eachcol(df2)) == [Int, Int, Int]
end

@testset "scalar on assignment side" begin
    df = DataFrame(rand(2, 3), :auto)
    @test_throws MethodError df[1, 1] .= df[1, 1] .- df[1, 1]
    df[1, 1:1] .= df[1, 1] .- df[1, 1]
    @test df[1, 1] == 0
    @test_throws MethodError df[1, 2] .-= df[1, 2]
    df[1:1, 2] .-= df[1, 2]
    @test df[1, 2] == 0
end

@testset "nothing test" begin
    X = DataFrame(Any[1 2; 3 4], :auto)
    X .= nothing
    @test (X .== nothing) == DataFrame(trues(2, 2), :auto)

    X = DataFrame([1 2; 3 4], :auto)
    @test_throws MethodError X .= nothing
    @test X == DataFrame([1 2; 3 4], :auto)

    X = DataFrame([1 2; 3 4], :auto)
    foreach(i -> X[!, i] .= nothing, axes(X, 2))
    @test (X .== nothing) == DataFrame(trues(2, 2), :auto)
end

@testset "aliasing test" begin
    df = DataFrame(x=[1, 2])
    y = view(df.x, [2, 1])
    df .= y
    @test df.x == [2, 1]

    df = DataFrame(x=[1, 2])
    y = view(df.x, [2, 1])
    dfv = view(df, :, :)
    dfv .= y
    @test df.x == [2, 1]

    df = DataFrame(x=2, y=1, z=1)
    dfr = df[1, :]
    y = view(df.x, 1)
    dfr .= 2 .* y
    @test Vector(dfr) == [4, 4, 4]

    df = DataFrame(x=[1, 2], y=[11, 12])
    df2 = DataFrame()
    df2.x = [-1, -2]
    df2.y = df.x
    df3 = copy(df2)
    df .= df2
    @test df == df3

    Random.seed!(1234)
    for i in 1:10
        df1 = DataFrame(rand(100, 100), :auto)
        df2 = copy(df1)
        for i in 1:100
            df2[!, rand(1:100)] = df1[!, i]
        end
        df3 = copy(df2)
        df1 .= df2
        @test df1 == df3
        @test df2 != df3
    end

    for i in 1:10
        df1 = DataFrame(rand(100, 100), :auto)
        df2 = copy(df1)
        for i in 1:100
            df2[!, rand(1:100)] = df1[!, i]
        end
        df3 = copy(df2)
        df1 .= view(df2, :, :)
        @test df1 == df3
        @test df2 != df3
    end

    for i in 1:10
        df1 = DataFrame(rand(100, 100), :auto)
        df2 = copy(df1)
        for i in 1:100
            df2[!, rand(1:100)] = df1[!, i]
        end
        df3 = copy(df2)
        view(df1, :, :) .= df2
        @test df1 == df3
        @test df2 != df3
    end

    for i in 1:10
        df1 = DataFrame(rand(100, 100), :auto)
        df2 = copy(df1)
        df3 = copy(df1)
        for i in 1:100
            df2[!, rand(1:100)] = df1[!, i]
            df3[!, rand(1:100)] = df1[!, i]
        end
        df6 = copy(df2)
        df7 = copy(df3)
        df4 = DataFrame(sin.(df1[1, 1] .+ copy(df1[!, 1]) .+ Matrix(df2) ./ Matrix(df3)), names(df3))
        df5 = sin.(view(df1, 1, 1) .+ df1[!, 1] .+ df2 ./ df3)
        df1 .= sin.(view(df1, 1, 1) .+ df1[!, 1] .+ df2 ./ df3)
        @test df1 == df4 == df5
        @test df2 != df6
        @test df3 != df7
    end

    for i in 1:10
        df1 = DataFrame(rand(100, 100), :auto)
        df2 = copy(df1)
        df3 = copy(df1)
        for i in 1:100
            df2[!, rand(1:100)] = df1[!, i]
            df3[!, rand(1:100)] = df1[!, i]
        end
        df6 = copy(df2)
        df7 = copy(df3)
        df4 = DataFrame(sin.(df1[1, 1] .+ copy(df1[!, 1]) .+ Matrix(df2) ./ Matrix(df3)), names(df3))
        df5 = sin.(view(df1, 1, 1) .+ df1[!, 1] .+ view(df2, :, :) ./ df3)
        df1 .= sin.(view(df1[!, 1], 1) .+ view(df1[!, 1], :) .+ df2 ./ view(df3, :, :))
        @test df1 == df4 == df5
        @test df2 != df6
        @test df3 != df7
    end

    for i in 1:10
        df1 = DataFrame(rand(100, 100), :auto)
        df2 = copy(df1)
        df3 = copy(df1)
        for i in 1:100
            df2[!, rand(1:100)] = df1[!, i]
            df3[!, rand(1:100)] = df1[!, i]
        end
        df6 = copy(df2)
        df7 = copy(df3)
        df4 = DataFrame(sin.(df1[1, 1] .+ copy(df1[!, 1]) .+ Matrix(df2) ./ Matrix(df3)), names(df3))
        df5 = sin.(view(df1, 1, 1) .+ df1[!, 1] .+ view(df2, :, :) ./ df3)
        view(df1, :, :) .= sin.(view(df1[!, 1], 1) .+ view(df1[!, 1], :) .+ df2 ./ view(df3, :, :))
        @test df1 == df4 == df5
        @test df2 != df6
        @test df3 != df7
    end
end

@testset "@. test" begin
    df = DataFrame(rand(2, 3), :auto)
    sdf = view(df, 1:1, :)
    dfm = Matrix(df)
    sdfm = Matrix(sdf)

    r1 = @. (df + sdf + 5) / sdf
    @test r1 isa DataFrame

    @. df = sin(sdf / (df + 1))
    @. dfm = sin(sdfm / (dfm + 1))
    @test df == DataFrame(dfm, names(df))
end

@testset "test common cases" begin
    m = rand(1000, 10)
    df = DataFrame(m, :auto)
    @test df .+ 1 == DataFrame(m .+ 1, names(df))
    @test df .+ transpose(1:10) == DataFrame(m .+ transpose(1:10), names(df))
    @test df .+ (1:1000) == DataFrame(m .+ (1:1000), names(df))
    @test df .+ m == DataFrame(m .+ m, names(df))
    @test m .+ df == DataFrame(m .+ m, names(df))
    @test df .+ df == DataFrame(m .+ m, names(df))

    df .+= 1
    m .+= 1
    @test df == DataFrame(m, names(df))
    df .+= transpose(1:10)
    m .+= transpose(1:10)
    @test df == DataFrame(m, names(df))
    df .+= (1:1000)
    m .+= (1:1000)
    @test df == DataFrame(m, names(df))
    df .+= df
    m .+= m
    @test df == DataFrame(m, names(df))
    df2 = copy(df)
    m2 = copy(m)
    df .+= df .+ df2 .+ m2 .+ 1
    m .+= m .+ df2 .+ m2 .+ 1
    @test df == DataFrame(m, names(df))
end

@testset "data frame only on left hand side broadcasting assignment" begin
    Random.seed!(1234)

    m = rand(3, 4);
    m2 = copy(m);
    m3 = copy(m);
    df = DataFrame(a=view(m, :, 1), b=view(m, :, 1),
                   c=view(m, :, 1), d=view(m, :, 1), copycols=false);
    df2 = copy(df)
    mdf = Matrix(df)

    @test m .+ df == m2 .+ df
    @test Matrix(m .+ df) == m .+ mdf
    @test sin.(m .+ df) .+ 1 .+ m2 == sin.(m2 .+ df) .+ 1 .+ m
    @test Matrix(m .+ df ./ 2 .* df2) == m .+ mdf ./ 2 .* mdf

    m2 .+= df .+ 1 ./ df2
    m .+= df .+ 1 ./ df2
    @test m2 == m
    for col in eachcol(df)
        @test col == m[:, 1]
    end
    for col in eachcol(df2)
        @test col == m3[:, 1]
    end

    m = rand(3, 4);
    m2 = copy(m);
    m3 = copy(m);
    df = view(DataFrame(a=view(m, :, 1), b=view(m, :, 1),
                        c=view(m, :, 1), d=view(m, :, 1), copycols=false),
              [3, 2, 1], :)
    df2 = copy(df)
    mdf = Matrix(df)

    @test m .+ df == m2 .+ df
    @test Matrix(m .+ df) == m .+ mdf
    @test sin.(m .+ df) .+ 1 .+ m2 == sin.(m2 .+ df) .+ 1 .+ m
    @test Matrix(m .+ df ./ 2 .* df2) == m .+ mdf ./ 2 .* mdf

    m2 .+= df .+ 1 ./ df2
    m .+= df .+ 1 ./ df2
    @test m2 == m
    for col in eachcol(df)
        @test col == m[3:-1:1, 1]
    end
    for col in eachcol(df2)
        @test col == m3[3:-1:1, 1]
    end
end

@testset "broadcasting with 3-dimensional object" begin
    y = zeros(4, 3, 2)
    df = DataFrame(ones(4, 3), :auto)
    @test_throws DimensionMismatch df .+ y
    @test_throws DimensionMismatch y .+ df
    @test_throws DimensionMismatch df .+= y
    y .+= df
    @test y == ones(4, 3, 2)
end

@testset "additional checks of post-! broadcasting rules" begin
    df = copy(refdf)
    v1 = df[!, 1]
    @test_throws MethodError df[CartesianIndex(1, 1)] .= 1
    @test_throws MethodError df[CartesianIndex(1, 1)] .= "d"
    @test_throws DimensionMismatch df[CartesianIndex(1, 1)] .= [1, 2]

    df = copy(refdf)
    v1 = df[!, 1]
    @test_throws MethodError df[1, 1] .= 1
    @test_throws MethodError df[1, 1] .= "d"
    @test_throws DimensionMismatch df[1, 1] .= [1, 2]

    df = copy(refdf)
    v1 = df[!, 1]
    @test_throws MethodError df[1, :x1] .= 1
    @test_throws MethodError df[1, :x1] .= "d"
    @test_throws DimensionMismatch df[1, :x1] .= [1, 2]

    df = copy(refdf)
    v1 = df[!, 1]
    @test_throws MethodError df[1, "x1"] .= 1
    @test_throws MethodError df[1, "x1"] .= "d"
    @test_throws DimensionMismatch df[1, "x1"] .= [1, 2]

    df = copy(refdf)
    v1 = df[!, 1]
    v2 = df[!, 2]
    df[1, 1:2] .= 'd'
    @test v1 == [100.0, 2.5, 3.5]
    @test v2 == [100.0, 5.5, 6.5]
    @test_throws MethodError df[1, 1:2] .= "d"
    @test v1 == [100.0, 2.5, 3.5]
    @test v2 == [100.0, 5.5, 6.5]
    df[1, 1:2] .= 'e':'f'
    @test v1 == [101.0, 2.5, 3.5]
    @test v2 == [102.0, 5.5, 6.5]
    @test_throws DimensionMismatch df[1, 1:2] .= ['d' 'd']
    @test v1 == [101.0, 2.5, 3.5]
    @test v2 == [102.0, 5.5, 6.5]

    df = copy(refdf)
    v1 = df[!, 1]
    df[:, 1] .= 'd'
    @test v1 == [100.0, 100.0, 100.0]
    @test_throws MethodError df[:, 1] .= "d"
    @test v1 == [100.0, 100.0, 100.0]
    @test_throws DimensionMismatch df[:, 1] .= [1 2 3]
    @test v1 == [100.0, 100.0, 100.0]

    df = copy(refdf)
    v1 = df[!, 1]
    df[:, :x1] .= 'd'
    @test v1 == [100.0, 100.0, 100.0]
    @test_throws MethodError df[:, :x1] .= "d"
    @test v1 == [100.0, 100.0, 100.0]
    @test_throws DimensionMismatch df[:, :x1] .= [1 2 3]
    @test v1 == [100.0, 100.0, 100.0]

    df = copy(refdf)
    v1 = df[!, 1]
    df[:, 1] .= 'd':'f'
    @test v1 == [100.0, 101.0, 102.0]
    @test_throws MethodError df[:, 1] .= ["d", "e", "f"]
    @test v1 == [100.0, 101.0, 102.0]

    df = copy(refdf)
    v1 = df[!, 1]
    v2 = df[!, 2]
    df[:, 1:2] .= 'd'
    @test v1 == [100.0, 100.0, 100.0]
    @test v2 == [100.0, 100.0, 100.0]
    @test_throws MethodError df[:, 1:2] .= "d"
    @test v1 == [100.0, 100.0, 100.0]
    @test v2 == [100.0, 100.0, 100.0]
    @test_throws DimensionMismatch df[:, 1:2] .= [1 2 3]
    @test v1 == [100.0, 100.0, 100.0]
    @test v2 == [100.0, 100.0, 100.0]

    df = copy(refdf)
    v1 = df[!, 1]
    v2 = df[!, 2]
    df[:, 1:2] .= 'd':'f'
    @test v1 == [100.0, 101.0, 102.0]
    @test v2 == [100.0, 101.0, 102.0]
    @test_throws MethodError df[:, 1:2] .= ["d", "e", "f"]
    @test v1 == [100.0, 101.0, 102.0]
    @test v2 == [100.0, 101.0, 102.0]

    df = copy(refdf)
    v1 = df[!, 1]
    v2 = df[!, 2]
    df[:, 1:2] .= permutedims('d':'e')
    @test v1 == [100.0, 100.0, 100.0]
    @test v2 == [101.0, 101.0, 101.0]

    df = copy(refdf)
    v1 = df[!, 1]
    v2 = df[!, 2]
    df[:, 1:2] .= reshape('d':'i', 3, :)
    @test v1 == [100.0, 101.0, 102.0]
    @test v2 == [103.0, 104.0, 105.0]
    @test_throws DimensionMismatch df[:, 1:2] .= reshape('d':'i', 3, :, 1)
    @test v1 == [100.0, 101.0, 102.0]
    @test v2 == [103.0, 104.0, 105.0]

    df = copy(refdf)
    v1 = df[!, 1]
    v1′ = df[:, 1]
    df[!, 1] .= 100.0
    @test df.x1 == [100.0, 100.0, 100.0]
    @test v1 == v1′
    df[!, 1] .= 'd'
    @test df.x1 == ['d', 'd', 'd']
    @test v1 == v1′
    @test_throws DimensionMismatch df[!, 1] .= [1 2 3]
    @test df.x1 == ['d', 'd', 'd']
    @test v1 == v1′

    df = copy(refdf)
    v1 = df[!, 1]
    v1′ = df[:, 1]
    df[!, :x1] .= 100.0
    @test df.x1 == [100.0, 100.0, 100.0]
    @test v1 == v1′
    df[!, :x1] .= 'd'
    @test df.x1 == ['d', 'd', 'd']
    @test v1 == v1′
    @test_throws DimensionMismatch df[!, :x1] .= [1 2 3]
    @test df.x1 == ['d', 'd', 'd']
    @test v1 == v1′

    df = copy(refdf)
    df[!, :newcol] .= 100.0
    @test df.newcol == [100.0, 100.0, 100.0]
    @test df[:, 1:end-1] == refdf

    df = copy(refdf)
    df[!, "newcol"] .= 100.0
    @test df.newcol == [100.0, 100.0, 100.0]
    @test df[:, 1:end-1] == refdf

    df = copy(refdf)
    df[!, :newcol] .= 'd'
    @test df.newcol == ['d', 'd', 'd']
    @test df[:, 1:end-1] == refdf

    df = copy(refdf)
    df[!, "newcol"] .= 'd'
    @test df.newcol == ['d', 'd', 'd']
    @test df[:, 1:end-1] == refdf

    df = copy(refdf)
    @test_throws DimensionMismatch df[!, :newcol] .= [1 2 3]
    @test df == refdf

    df = copy(refdf)
    @test_throws DimensionMismatch df[!, "newcol"] .= [1 2 3]
    @test df == refdf

    df = copy(refdf)
    @test_throws ArgumentError df[!, 10] .= 'a'
    @test df == refdf
    @test_throws ArgumentError df[!, 10] .= [1, 2, 3]
    @test df == refdf
    @test_throws ArgumentError df[!, 10] .= [1 2 3]
    @test df == refdf

    df = copy(refdf)
    df[!, 1:2] .= 'a'
    @test Matrix(df) == ['a'  'a'  7.5  10.5  13.5
                         'a'  'a'  8.5  11.5  14.5
                         'a'  'a'  9.5  12.5  15.5]

    df = copy(refdf)
    v1 = df[!, 1]
    df.x1 .= 'd'
    @test v1 == [100.0, 100.0, 100.0]
    @test_throws MethodError df[:, 1] .= "d"
    @test v1 == [100.0, 100.0, 100.0]
    @test_throws DimensionMismatch df[:, 1] .= [1 2 3]
    @test v1 == [100.0, 100.0, 100.0]

    df = copy(refdf)
    @test_throws ArgumentError df.newcol .= 'd'
    @test df == refdf

    df = view(copy(refdf), :, :)
    v1 = df[!, 1]
    @test_throws MethodError df[CartesianIndex(1, 1)] .= 1
    @test_throws MethodError df[CartesianIndex(1, 1)] .= "d"
    @test_throws DimensionMismatch df[CartesianIndex(1, 1)] .= [1, 2]

    df = view(copy(refdf), :, :)
    v1 = df[!, 1]
    @test_throws MethodError df[1, 1] .= 1
    @test_throws MethodError df[1, 1] .= "d"
    @test_throws DimensionMismatch df[1, 1] .= [1, 2]

    df = view(copy(refdf), :, :)
    v1 = df[!, 1]
    @test_throws MethodError df[1, :x1] .= 1
    @test_throws MethodError df[1, :x1] .= "d"
    @test_throws DimensionMismatch df[1, :x1] .= [1, 2]

    df = view(copy(refdf), :, :)
    v1 = df[!, 1]
    v2 = df[!, 2]
    df[1, 1:2] .= 'd'
    @test v1 == [100.0, 2.5, 3.5]
    @test v2 == [100.0, 5.5, 6.5]
    @test_throws MethodError df[1, 1:2] .= "d"
    @test v1 == [100.0, 2.5, 3.5]
    @test v2 == [100.0, 5.5, 6.5]
    df[1, 1:2] .= 'e':'f'
    @test v1 == [101.0, 2.5, 3.5]
    @test v2 == [102.0, 5.5, 6.5]
    @test_throws DimensionMismatch df[1, 1:2] .= ['d' 'd']
    @test v1 == [101.0, 2.5, 3.5]
    @test v2 == [102.0, 5.5, 6.5]

    df = view(copy(refdf), :, :)
    v1 = df[!, 1]
    df[:, 1] .= 'd'
    @test v1 == [100.0, 100.0, 100.0]
    @test_throws MethodError df[:, 1] .= "d"
    @test v1 == [100.0, 100.0, 100.0]
    @test_throws DimensionMismatch df[:, 1] .= [1 2 3]
    @test v1 == [100.0, 100.0, 100.0]

    df = view(copy(refdf), :, :)
    v1 = df[!, 1]
    df[:, :x1] .= 'd'
    @test v1 == [100.0, 100.0, 100.0]
    @test_throws MethodError df[:, :x1] .= "d"
    @test v1 == [100.0, 100.0, 100.0]
    @test_throws DimensionMismatch df[:, :x1] .= [1 2 3]
    @test v1 == [100.0, 100.0, 100.0]

    df = view(copy(refdf), :, :)
    v1 = df[!, 1]
    df[:, 1] .= 'd':'f'
    @test v1 == [100.0, 101.0, 102.0]
    @test_throws MethodError df[:, 1] .= ["d", "e", "f"]
    @test v1 == [100.0, 101.0, 102.0]

    df = view(copy(refdf), :, :)
    v1 = df[!, 1]
    v2 = df[!, 2]
    df[:, 1:2] .= 'd'
    @test v1 == [100.0, 100.0, 100.0]
    @test v2 == [100.0, 100.0, 100.0]
    @test_throws MethodError df[:, 1:2] .= "d"
    @test v1 == [100.0, 100.0, 100.0]
    @test v2 == [100.0, 100.0, 100.0]
    @test_throws DimensionMismatch df[:, 1:2] .= [1 2 3]
    @test v1 == [100.0, 100.0, 100.0]
    @test v2 == [100.0, 100.0, 100.0]

    df = view(copy(refdf), :, :)
    v1 = df[!, 1]
    v2 = df[!, 2]
    df[:, 1:2] .= 'd':'f'
    @test v1 == [100.0, 101.0, 102.0]
    @test v2 == [100.0, 101.0, 102.0]
    @test_throws MethodError df[:, 1:2] .= ["d", "e", "f"]
    @test v1 == [100.0, 101.0, 102.0]
    @test v2 == [100.0, 101.0, 102.0]

    df = view(copy(refdf), :, :)
    v1 = df[!, 1]
    v2 = df[!, 2]
    df[:, 1:2] .= permutedims('d':'e')
    @test v1 == [100.0, 100.0, 100.0]
    @test v2 == [101.0, 101.0, 101.0]

    df = view(copy(refdf), :, :)
    v1 = df[!, 1]
    v2 = df[!, 2]
    df[:, 1:2] .= reshape('d':'i', 3, :)
    @test v1 == [100.0, 101.0, 102.0]
    @test v2 == [103.0, 104.0, 105.0]
    @test_throws DimensionMismatch df[:, 1:2] .= reshape('d':'i', 3, :, 1)
    @test v1 == [100.0, 101.0, 102.0]
    @test v2 == [103.0, 104.0, 105.0]

    df = view(copy(refdf), :, :)
    @test_throws ArgumentError df[!, 1] .= 100.0
    @test df == refdf

    df = view(copy(refdf), :, :)
    @test_throws ArgumentError df[!, :x1] .= 100.0
    @test df == refdf

    df = view(copy(refdf), :, :)
    @test_throws ArgumentError df[!, :newcol] .= 100.0
    @test df == refdf

    df = view(copy(refdf), :, :)
    @test_throws ArgumentError df[!, 10] .= 'a'
    @test df == refdf
    @test_throws ArgumentError df[!, 10] .= [1, 2, 3]
    @test df == refdf
    @test_throws ArgumentError df[!, 10] .= [1 2 3]
    @test df == refdf

    df = view(copy(refdf), :, :)
    @test_throws ArgumentError df[!, 1:2] .= 'a'
    @test df == refdf

    df = view(copy(refdf), :, :)
    v1 = df[!, 1]
    df.x1 .= 'd'
    @test v1 == [100.0, 100.0, 100.0]
    @test_throws MethodError df[:, 1] .= "d"
    @test v1 == [100.0, 100.0, 100.0]
    @test_throws DimensionMismatch df[:, 1] .= [1 2 3]
    @test v1 == [100.0, 100.0, 100.0]

    df = view(copy(refdf), :, :)
    @test_throws ArgumentError df.newcol .= 'd'
    @test df == refdf
end

@testset "DataFrameRow getproperty broadcasted assignment" begin
    df = DataFrame(a=[[1, 2], [3, 4]], b=[[5, 6], [7, 8]])
    dfr = df[1, :]
    dfr.a .= 10
    @test df == DataFrame(a=[[10, 10], [3, 4]], b=[[5, 6], [7, 8]])
    @test_throws MethodError dfr.a .= ["a", "b"]

    df = DataFrame(a=[[1, 2], [3, 4]], b=[[5, 6], [7, 8]])
    dfr = df[1, 1:1]
    dfr.a .= 10
    @test df == DataFrame(a=[[10, 10], [3, 4]], b=[[5, 6], [7, 8]])
    @test_throws MethodError dfr.a .= ["a", "b"]

    df = DataFrame(a=[[1, 2], [3, 4]], b=[[5, 6], [7, 8]])
    dfr = df[1, :]
    dfr."a" .= 10
    @test df == DataFrame(a=[[10, 10], [3, 4]], b=[[5, 6], [7, 8]])
    @test_throws MethodError dfr."a" .= ["a", "b"]

    df = DataFrame(a=[[1, 2], [3, 4]], b=[[5, 6], [7, 8]])
    dfr = df[1, 1:1]
    dfr."a" .= 10
    @test df == DataFrame(a=[[10, 10], [3, 4]], b=[[5, 6], [7, 8]])
    @test_throws MethodError dfr."a" .= ["a", "b"]
end

@testset "make sure that : is in place and ! allocates" begin
    df = DataFrame(a = [1, 2, 3])
    a = df.a
    df[:, :a] .+= 1
    @test a == [2, 3, 4]
    @test df.a === a
    df[!, :a] .+= 1
    @test a == [2, 3, 4]
    @test df.a == [3, 4, 5]
    @test df.a !== a

    df = DataFrame(a = [1, 2, 3])
    a = df.a
    df[:, "a"] .+= 1
    @test a == [2, 3, 4]
    @test df.a === a
    df[!, "a"] .+= 1
    @test a == [2, 3, 4]
    @test df.a == [3, 4, 5]
    @test df.a !== a
end

@testset "add new correct rules for df[row, col] .= v broadcasting" begin
    for v in [:a, "a"]
        df = DataFrame(a=1)
        @test_throws MethodError df[1, 1] .= 10
        @test_throws MethodError df[1, v] .= 10
        @test_throws MethodError df[CartesianIndex(1, 1)] .= 10
        df = DataFrame(a=[[1, 2, 3]])
        df[1, 1] .= 10
        @test df == DataFrame(a=[[10, 10, 10]])
        df[1, v] .= 100
        @test df == DataFrame(a=[[100, 100, 100]])
        df[CartesianIndex(1, 1)] .= 1000
        @test df == DataFrame(a=[[1000, 1000, 1000]])
    end
end

@testset "broadcasting into df[!, cols]" begin
    for selector in [1:2, Between(:x1, :x2), Not(r"x3"), [:x1, :x2],
                     ["x1", "x2"], Between("x1", "x2")]
        df = DataFrame(x1=1:3, x2=4:6)
        df[!, selector] .= "a"
        @test df == DataFrame(fill("a", 3, 2), :auto)
        @test df.x1 !== df.x2

        df = DataFrame(x1=1:3, x2=4:6)
        df[!, selector] .= Ref((a=1, b=2))
        @test df == DataFrame(fill((a=1, b=2), 3, 2), :auto)
        @test df.x1 !== df.x2

        df = DataFrame(x1=1:3, x2=4:6)
        df[!, selector] .= ["a" "b"]
        @test df == DataFrame(["a" "b"
                               "a" "b"
                               "a" "b"], :auto)
        @test df.x1 !== df.x2

        df = DataFrame(x1=1:3, x2=4:6)
        df[!, selector] .= ["a", "b", "c"]
        @test df == DataFrame(["a" "a"
                               "b" "b"
                               "c" "c"], :auto)
        @test df.x1 !== df.x2

        df = DataFrame(x1=1:3, x2=4:6)
        df[!, selector] .= categorical(["a"])
        @test df == DataFrame(["a" "a"
                               "a" "a"
                               "a" "a"], :auto)
        @test df.x1 isa CategoricalVector
        @test df.x2 isa CategoricalVector
        @test df.x1 !== df.x2

        df = DataFrame(x1=1:3, x2=4:6)
        df[!, selector] .= DataFrame(["a" "b"], :auto)
        @test df == DataFrame(["a" "b"
                               "a" "b"
                               "a" "b"], :auto)
        @test df.x1 !== df.x2

        df = DataFrame(x1=1:3, x2=4:6)
        df[!, selector] .= DataFrame(["a" "d"
                                      "b" "e"
                                      "c" "f"], :auto)
        @test df == DataFrame(["a" "d"
                               "b" "e"
                               "c" "f"], :auto)
        @test df.x1 !== df.x2

        df = DataFrame(x1=1:3, x2=4:6)
        df[!, selector] .= ["a" "d"
                            "b" "e"
                            "c" "f"]
        @test df == DataFrame(["a" "d"
                               "b" "e"
                               "c" "f"], :auto)
        @test df.x1 !== df.x2

        df = DataFrame(x1=1:3, x2=4:6, x3=1)
        df[!, selector] .= "a"
        @test df == DataFrame(["a" "a" 1
                               "a" "a" 1
                               "a" "a" 1], :auto)
        @test df.x1 !== df.x2

        df = DataFrame(x1=1:3, x2=4:6, x3=1)
        df[!, selector] .= Ref((a=1, b=2))
        @test df[:, 1:2] == DataFrame(fill((a=1, b=2), 3, 2), :auto)
        @test df[:, 3] == [1, 1, 1]
        @test df.x1 !== df.x2

        df = DataFrame(x1=1:3, x2=4:6, x3=1)
        df[!, selector] .= ["a" "b"]
        @test df == DataFrame(["a" "b" 1
                               "a" "b" 1
                               "a" "b" 1], :auto)
        @test df.x1 !== df.x2

        df = DataFrame(x1=1:3, x2=4:6, x3=1)
        df[!, selector] .= ["a", "b", "c"]
        @test df == DataFrame(["a" "a" 1
                               "b" "b" 1
                               "c" "c" 1], :auto)
        @test df.x1 !== df.x2

        df = DataFrame(x1=1:3, x2=4:6, x3=1)
        df[!, selector] .= categorical(["a"])
        @test df == DataFrame(["a" "a" 1
                               "a" "a" 1
                               "a" "a" 1], :auto)
        @test df.x1 isa CategoricalVector
        @test df.x2 isa CategoricalVector
        @test df.x1 !== df.x2

        df = DataFrame(x1=1:3, x2=4:6, x3=1)
        df[!, selector] .= DataFrame(["a" "b"], :auto)
        @test df == DataFrame(["a" "b" 1
                               "a" "b" 1
                               "a" "b" 1], :auto)
        @test df.x1 !== df.x2

        df = DataFrame(x1=1:3, x2=4:6, x3=1)
        df[!, selector] .= DataFrame(["a" "d"
                                      "b" "e"
                                      "c" "f"], :auto)
        @test df == DataFrame(["a" "d" 1
                               "b" "e" 1
                               "c" "f" 1], :auto)
        @test df.x1 !== df.x2

        df = DataFrame(x1=1:3, x2=4:6, x3=1)
        df[!, selector] .= ["a" "d"
                            "b" "e"
                            "c" "f"]
        @test df == DataFrame(["a" "d" 1
                               "b" "e" 1
                               "c" "f" 1], :auto)
        @test df.x1 !== df.x2
    end

    df = DataFrame(x1=1:3, x2=4:6)
    @test_throws ArgumentError df[!, [:x1, :x3]] .= "a"
end

@testset "broadcasting over heterogenous columns" begin
    df = DataFrame(x = [1, 1.0, big(1), "1"])
    f_identity(x) = x
    @test df == f_identity.(df)
end

@testset "@views on df[!, col]" begin
    df = DataFrame(ones(3, 4), :auto)
    @views df[!, 1] .+= 1
    @test df[!, 1] == [2.0, 2.0, 2.0]
    @views df[:, 2] .= df[!, 4] .+ df[!, 3]
    @test df[!, 2] == [2.0, 2.0, 2.0]

    # make sure we do not mess with maybeview
    @test @views typeof(df[!, 1:2]) <: SubDataFrame
end

@testset "broadcasting of df[:, col] = value" begin
    df = DataFrame(ones(3, 4), :auto)
    z = ["a", "b", "c"]
    df[:, :z] .= z
    @test df.z == z
    @test df.z !== z
    @test_throws ArgumentError df[:, 6] .= z
    @test_throws MethodError df[:, 1] .= z

    df = DataFrame(ones(3, 4), :auto)
    z = "abc"
    df[:, :z] .= z
    @test df.z == fill("abc", 3)
    @test_throws ArgumentError df[:, 6] .= z
    @test_throws MethodError df[:, 1] .= z

    df = DataFrame(ones(3, 4), :auto)
    z = fill("abc", 1, 1, 1)
    @test_throws DimensionMismatch df[:, :z] .= z

    df = DataFrame(ones(3, 4), :auto)
    z = ["a", "b", "c"]
    df[:, "z"] .= z
    @test df.z == z
    @test df.z !== z

    df = DataFrame(ones(3, 4), :auto)
    z = "abc"
    df[:, "z"] .= z
    @test df.z == fill("abc", 3)

    df = DataFrame(ones(3, 4), :auto)
    z = fill("abc", 1, 1, 1)
    @test_throws DimensionMismatch df[:, "z"] .= z
end

end # module


text:
"""
Main module controlling displaying segmentations image and data

"""
module SegmentationDisplay
export coordinateDisplay
export passDataForScrolling

using ModernGL, GLFW,  ..PrepareWindow,  ..TextureManag, ..OpenGLDisplayUtils,  ..ForDisplayStructs, ..Uniforms,  ..DisplayWords, Dictionaries
using  ..ReactingToInput, Rocket, Setfield, Logging,  ..ShadersAndVerticiesForText,FreeTypeAbstraction, ..DisplayWords,  ..DataStructs,  ..StructsManag

using DrWatson
@quickactivate "MedEye3d"

#holds actor that is main structure that process inputs from GLFW and reacts to it
mainActor = sync(ActorWithOpenGlObjects())
#collecting all subsciptions  to be able to clean all later
subscriptions = []

"""
coordinating displaying - sets needed constants that are storeds in  forDisplayConstants; and configures interactions from GLFW events
listOfTextSpecs - holds required data needed to initialize textures
keeps also references to needed ..Uniforms etc.
windowWidth::Int,windowHeight::Int - GLFW window dimensions
fractionOfMainIm - how much of width should be taken by the main image
heightToWithRatio - needed for proper display of main texture - so it would not be stretched ...
"""
function coordinateDisplay(listOfTextSpecsPrim::Vector{TextureSpec}
                        ,fractionOfMainIm::Float32
                        ,dataToScrollDims::DataToScrollDims=DataToScrollDims()#stores additional data about full dimensions of scrollable dat - this is necessery for switching slicing plane orientation efficiently
                        ,windowWidth::Int=1200
                        ,windowHeight::Int= Int(round(windowWidth*fractionOfMainIm))
                        ,textTexturewidthh::Int32=Int32(2000)
                        ,textTextureheightt::Int32= Int32( round((windowHeight/(windowWidth*(1-fractionOfMainIm)) ))*textTexturewidthh)
                        ,windowControlStruct::WindowControlStruct=WindowControlStruct())

   #in case we are recreating all we need to destroy old textures ... generally simplest is destroy window
    if( typeof(mainActor.actor.mainForDisplayObjects.window)== GLFW.Window  )
        cleanUp()
    end#if
   #setting number to texture that will be needed in shader configuration
   listOfTextSpecs= map(x->setproperties(x[2],(whichCreated=x[1])),enumerate(listOfTextSpecsPrim))

   #calculations of necessary constants needed to calculate window size , mouse position ...
   calcDimStruct= CalcDimsStruct(windowWidth=windowWidth
                  ,windowHeight=windowHeight
                  ,fractionOfMainIm=fractionOfMainIm
                  ,wordsImageQuadVert= ShadersAndVerticiesForText.getWordsVerticies(fractionOfMainIm)
                  ,wordsQuadVertSize= sizeof( ShadersAndVerticiesForText.getWordsVerticies(fractionOfMainIm))
                  ,textTexturewidthh=textTexturewidthh
                  ,textTextureheightt=textTextureheightt ) |>
   (calcDim)-> getHeightToWidthRatio(calcDim,dataToScrollDims )|>
   (calcDim)-> getMainVerticies(calcDim)

   subscribe!(of(calcDimStruct),mainActor )

 #creating window and event listening loop
    window,vertex_shader,fragment_shader ,shader_program,stopListening,vbo,ebo,fragment_shader_words,vbo_words,shader_program_words,gslsStr =  PrepareWindow.displayAll(listOfTextSpecs,calcDimStruct )

    # than we set those ..Uniforms, open gl types and using data from arguments  to fill texture specifications
    mainImageUnifs,listOfTextSpecsMapped= assignUniformsAndTypesToMasks(listOfTextSpecs,shader_program,windowControlStruct)

    @info "listOfTextSpecsMapped" listOfTextSpecsMapped
    #initializing object that holds data reqired for interacting with opengl
    initializedTextures =  initializeTextures(listOfTextSpecsMapped,calcDimStruct)

    numbDict = filter(x-> x.numb>=0,initializedTextures) |>
    (filtered)-> Dictionary(map(it->it.numb,filtered),collect(eachindex(filtered))) # a way for fast query using assigned numbers

    forDispObj =  forDisplayObjects(
            listOfTextSpecifications=initializedTextures
            ,window= window
            ,vertex_shader= vertex_shader
            ,fragment_shader= fragment_shader
            ,shader_program= shader_program
            ,stopListening= stopListening
            ,vbo= vbo[]
            ,ebo= ebo[]
            ,mainImageUniforms= mainImageUnifs
            ,TextureIndexes= Dictionary(map(it->it.name,initializedTextures),collect(eachindex(initializedTextures)))
            ,numIndexes= numbDict
            ,gslsStr=gslsStr
            ,windowControlStruct=windowControlStruct
   )

    #finding some texture that can be modifid and set as one active for modifications

       mainActor.actor.textureToModifyVec = filter(it->it.isEditable ,initializedTextures)

    #in order to clean up all resources while closing
    GLFW.SetWindowCloseCallback(window, (_) -> cleanUp())

    #wrapping the Open Gl and GLFW objects into an observable and passing it to the actor
    forDisplayConstantObesrvable = of(forDispObj)
    subscribe!(forDisplayConstantObesrvable, mainActor) # configuring
    #passing for text display object
    forTextDispStruct = prepareForDispStruct(length(initializedTextures)
                                            ,fragment_shader_words
                                            ,vbo_words
                                            ,shader_program_words
                                            ,window
                                            ,textTexturewidthh
                                            ,textTextureheightt
                                            ,forDispObj)

    subscribe!(of(forTextDispStruct),mainActor )

    registerInteractions()#passing needed subscriptions from GLFW

end #coordinateDisplay

"""
is used to pass into the actor data that will be used for scrolling
onScrollData - struct holding between others list of tuples where first is the name of the texture that we provided and second is associated data (3 dimensional array of appropriate type)
"""
function passDataForScrolling(onScrollData::FullScrollableDat)
    #wrapping the data into an observable and passing it to the actor
    forScrollData = of(onScrollData)
    subscribe!(forScrollData, mainActor)
end

"""
enables updating just a single slice that is displayed - do not change what will happen after scrolling
one need to pass data to actor in
listOfDataAndImageNames - struct holding  tuples where first entry in tuple is name of texture given in the setup and second is 2 dimensional aray of appropriate type with image data
sliceNumber - the number to which we set slice in order to later start scrolling the scroll data from this point
"""
function updateSingleImagesDisplayed( listOfDataAndImageNames::SingleSliceDat)
    forDispData = of(listOfDataAndImageNames)
    subscribe!(forDispData, mainActor)

end #updateSingleImagesDisplayed

 """
is using the actor that is instantiated in this module and connects it to GLFW context
by invoking appropriate registering functions and passing to it to the main Actor controlling input
"""
function registerInteractions()
    subscriptionsInner = subscribeGLFWtoActor(mainActor)
    for el in subscriptionsInner
        push!(subscriptions,el)
    end #for

end

"""
Preparing ForWordsDispStruct that will be needed for proper displaying of texts
    numberOfActiveTextUnits - number of textures already used - so we we will know what is still free
    fragment_shader_words - reference to fragment shader used to display text
    vbo_words - vertex buffer object used to display words
    shader_program_words - shader program associated with displaying text
    widthh, heightt - size of the texture - the bigger the higher resolution, but higher computation cost

return prepared for displayStruct
"""
function prepareForDispStruct(numberOfActiveTextUnits::Int
                            ,fragment_shader_words::UInt32
                            ,vbo_words::Base.RefValue{UInt32}
                            ,shader_program_words::UInt32
                            ,window
                            ,widthh::Int32 =Int32(1)
                            ,heightt::Int32=Int32(1)
                            ,forDispObj::forDisplayObjects=forDisplayObjects()
                            ) ::ForWordsDispStruct

      res =  ForWordsDispStruct(
            fontFace = FreeTypeAbstraction.findfont("hack";  additional_fonts= datadir("fonts"))
            ,textureSpec = createTextureForWords(numberOfActiveTextUnits
                                                 ,widthh
                                                 ,heightt
                                                 ,getProperGL_TEXTURE(numberOfActiveTextUnits+1) )
            ,fragment_shader_words= fragment_shader_words
            ,vbo_words=vbo_words
            ,shader_program_words=shader_program_words
         )

    return res
end#prepereForDispStruct

"""
In order to properly close displayer we need to :
 remove buffers that wer use
 remove shaders
 remove all textures
 unsubscibe all of the subscriptions to the mainActor
 finalize main actor and reinstantiate it
 close GLFW window
"""
function cleanUp()
    obj = mainActor.actor.mainForDisplayObjects
    glDeleteTextures(length(obj.listOfTextSpecifications), map(text->text.ID,obj.listOfTextSpecifications));
    glFlush()
    GLFW.DestroyWindow(obj.window)

    # glClearColor(0.0, 0.0, 0.1 , 1.0) # for a good begining
    # #first we unsubscribe and give couple seconds for processes to stop
    # for sub in subscriptions
    #     unsubscribe!(sub)
    # end # for
    # sleep(5)
    # obj = mainActor.actor.mainForDisplayObjects
    # #deleting textures
    # #destroying buffers
    # glDeleteBuffers(2,[obj.vbo,obj.ebo])
    # #detaching shaders
    # glDeleteShader(obj.fragment_shader);
    # glDeleteShader(obj.vertex_shader);
    # #destroying program
    # glDeleteProgram(obj.shader_program)
    # #finalizing and recreating main actor
end #cleanUp

end #SegmentationDisplay

ext: jl


text:
using YaoPastaQ, Yao
using PastaQ: MPS, productstate
using Test

@testset "YaoPastaQ.jl" begin
    @test genlist(chain(3, put(2 => X), repeat(X, 1:3))) ==  Any[("X", 2), ("X", 1), ("X", 2), ("X", 3)]
    @test genlist(chain(3, put(2 => Y), repeat(Y, 1:3), put(1:3 => chain(3, put(1=>X))))) == Any[("Y", 2), ("Y", 1), ("Y", 2), ("Y", 3), ("X", 1)]
    @test genlist(chain(3, put(1 => H), repeat(H, 1:3), put(1:3 => chain(3, put(1 => Z))))) == Any[("H", 1), ("H", 1), ("H", 2), ("H", 3), ("Z", 1)]
    @test genlist(chain(3, kron(X, Y, ConstGate.S))) == Any[("X", 1), ("Y", 2), ("Phase", 3)]

    testgate = chain(3, put(2 => X), put(3 => im*Y), repeat(H, 1:2), put(1:2 => chain(2, put(1 => X))), put(1 => shift(π/2)), put(1 => shift(π/4)), put(3=>Rx(π/3)), swap(1,2), control(1:2, 3=>X), control(1, 3=>Rz(π/3)), control(1, 2=>Y), control(1, 2:3=>SWAP))
    testgate2 = chain(4, control(1:3, 4=>X))

    @test genlist(testgate) == Any[("X", 2), ("iY", 3), ("H", 1), ("H", 2), ("X", 1), ("Phase", 1), ("π/8", 1), ("Rx", 3, (θ = 1.0471975511965976,)), ("SWAP", (1, 2)), ("Toffoli", (1, 2, 3)), ("CRz", (1, 3), (ϕ = 1.0471975511965976,)), ("CY", (1, 2)), ("Fredkin", (1, 2, 3))]
    @test genlist(testgate2) == Any[("CCCNOT", (1, 2, 3, 4))]
    @test nqubits(PastaQReg(3)) == 3
    @test PastaQReg(3) isa PastaQReg{MPS}
    @test measure(PastaQReg(3), 3) == [0 0 0; 0 0 0; 0 0 0]
    @test apply!(PastaQReg(3), chain(3, put(1=>X))) isa PastaQReg{MPS}
    @test PastaQReg(YaoBase.bit"0011") isa PastaQReg{MPS}
    @test fidelity(PastaQReg(3), PastaQReg(3)) == 1.0
    @test nactive(PastaQReg(YaoBase.bit"0011")) == 4
    @test PastaQReg(PastaQReg(3)) isa PastaQReg{MPS}
    @test PastaQReg(productstate(3)) isa PastaQReg{MPS}
end


text:
module QJuliaPipeCG

using QJuliaInterface
using QJuliaEnums
using QJuliaBlas
using QJuliaReduce
using QJuliaSolvers

using LinearAlgebra
using Printf

norm2    = QJuliaReduce.gnorm2
rdot     = QJuliaReduce.reDotProduct
cpy      = QJuliaBlas.cpy

fcgdebug = false

# References:
# P. Sanan, S.M. Schnepp, and D.A. May, "Pipelined, Flexible Krylov Subspace Methods,"
# SIAM Journal on Scientific Computing 2016 38:5, C441-C470,
# See also:
# S. Cools, E.F. Yetkin, E. Agullo, L. Giraud, W. Vanroose, "Analyzing the effect of local rounding error
# propagation on the maximal attainable accuracy of the pipelined Conjugate Gradients method",
# SIAM Journal on Matrix Analysis and Applications (SIMAX), 39(1):426–450, 2018.

function solver(x::AbstractArray, b::AbstractArray, Mat::Any, MatSloppy::Any, param::QJuliaSolvers.QJuliaSolverParam_qj, K::Function, extra_args...)

    is_preconditioned = param.inv_type_precondition != QJuliaEnums.QJULIA_INVALID_INVERTER

    solver_name = "PipeCG2"

    println("Running ", solver_name ," solver (solver precion ", param.dtype, " , sloppy precion ", param.dtype_sloppy, " )")

    if (param.maxiter == 0)
      if param.use_init_guess == false; x .=@. 0.0;end
      return
    end

    mixed = (param.dtype_sloppy != param.dtype)

    if mixed == true; println("Running mixed precision solver."); end

    ϵ = eps(param.dtype_sloppy)
    sqrteps = param.delta*sqrt(ϵ)

    Δcr = 0.0; Δcs = 0.0; Δcw = 0.0; Δcz = 0.0
    errr = 0.0; errrprev = 0.0; errs = 0.0; errw = 0.0; errz = 0.0
    replace = 0;totreplaces = 0

    local r_fp  = zeros(param.dtype, length(x))
    local z_fp  = zeros(param.dtype, length(x))
    local p_fp  = zeros(param.dtype, length(x))
    local w_fp  = zeros(param.dtype, length(x))
    local s_fp  = zeros(param.dtype, length(x))

    local r   = mixed == true ? zeros(param.dtype_sloppy, length(x)) : r_fp
    local z   = mixed == true ? zeros(param.dtype_sloppy, length(x)) : z_fp
    local p   = mixed == true ? zeros(param.dtype_sloppy, length(x)) : p_fp
    local w   = mixed == true ? zeros(param.dtype_sloppy, length(x)) : w_fp
    local s   = mixed == true ? zeros(param.dtype_sloppy, length(x)) : s_fp
    local n   = zeros(param.dtype_sloppy, length(x))

    if param.use_init_guess == true
      #r = b - Ax
      Mat(r_fp, x)
      r_fp .=@. b - r_fp
    else
      r_fp .=@. b
    end

    cpy(r, r_fp)

    norm2b = norm(b)
    rnorm  = norm(r_fp)

    MatSloppy(w, r)   #  w <- Au

    stop = rnorm*rnorm*param.tol*param.tol
    println(solver_name," : Initial (relative) residual ", rnorm / norm2b)

    # zero cycle
    γ      = rdot(r, r)
    δ      = rdot(w, r)

    MatSloppy(n, w)  #   n <- Am

    η  = δ
    α  = γ / η; β = 0.0

    z .=@. n           #  z <- n
    p .=@. r           #  p <- u
    s .=@. w           #  s <- w
    x .=@. x + α*p     #  x <- x + alpha * p
    w .=@. w - α*z     #  w <- w - alpha * z
    r .=@. r - α*s     #  r <- r - alpha * s

    rnorm  = norm(r)
    @printf("%s : first cycle residual: %1.15e \n", solver_name, rnorm/norm2b)

    k = 1; converged = false

    while (k < param.maxiter && converged == false)

      γold = γ; γ = rdot(r, r)
      τ     = rdot(s, r)
      δ     = rdot(w, r)
      rnorm = sqrt(γ)

      Σ  = sqrt(norm2(s))
      Ζ  = sqrt(norm2(z))

      MatSloppy(n, w)           #   n <- Am

      β = -τ / η
      η = δ - β*β*η
      αold = α; α = γ / η

      z .=@. n + β*z     #  z <- n + beta * z
      p .=@. r + β*p     #  p <- u + beta * p
      s .=@. w + β*s     #  s <- w + beta * s
      x .=@. x + α*p     #  x <- x + alpha * p
      w .=@. w - α*z     #  w <- w - alpha * z
      r .=@. r - α*s     #  r <- r - alpha * s

      Δcr = (2.0*αold*Σ)*ϵ
      Δcs = (2.0*β*Σ+2.0*αold*Ζ)*ϵ
      Δcw = (2.0*αold*Ζ)*ϵ
      Δcz = (2.0*β*Ζ)*ϵ

      if k == 1 || replace == 1
        println("(Re-)initialize reliable parameters..")
        errrprev = errr
        errr = Δcr
        errs = Δcs
        errw = Δcw
        errz = Δcz
        replace = 0
      else
        errrprev = errr
        errr = errr + αold*errs + Δcr
        errs = β*errs + errw + αold*errz + Δcs
        errw = errw + αold*errz + Δcw
        errz = β*errz + Δcz
      end

      # Check convergence:
      converged = ((rnorm*rnorm) < stop)
      @printf("%s: %d iteration, iter residual: %1.15e\n", solver_name, k, rnorm/norm2b)

      do_restart = (γ < 0.0) || ( (k > 1 && errrprev <= (sqrteps * sqrt(γold)) && errr > (sqrteps * sqrt(abs(γ)))))

      if do_restart == true
        println("Start reliable update...")
        Mat(r_fp,x)        #  r <- Ax - b
        r_fp .=@. b - r_fp
        norm2r = norm(r_fp)

        Mat(w_fp,r_fp)        #  w <- Ar
        cpy(r, r_fp)
        cpy(w,w_fp)

        cpy(p_fp,p)
        Mat(s_fp,p_fp)        #  s <- Ap
        Mat(z_fp,s_fp)        #  z <- As
        cpy(s,s_fp)
        cpy(z,z_fp)

        γ = γ < 0.0 ? rdot(r_fp, r_fp) : γ

        converged = (norm2r*norm2r < stop)

        @printf("True residual after update %1.15e (relative %1.15e, stop criterio %1.15e).\n", norm2r, norm2r/norm2b, stop)
        replace = 1;  totreplaces +=1
      end
      # Update iter index
      k += 1
    end # while
    @printf("Finish %s: %d iterations, total restarst: %d \n", solver_name, k, totreplaces)

end # solver

end # module QJuliaPipeCG2

max_stars_repo_name: alexstrel/QJulia

max_issues_repo_name: alexstrel/QJulia

max_forks_repo_name: alexstrel/QJulia


text:
export BicategoryRelations, FreeBicategoryRelations,
  AbelianBicategoryRelations, FreeAbelianBicategoryRelations,
  DistributiveBicategoryRelations,
  meet, join, top, bottom, plus, zero, coplus, cozero

import Base: join, zero

# Bicategory of relations
#########################

""" Theory of *bicategories of relations*

TODO: The 2-morphisms are missing.

References:

- Carboni & Walters, 1987, "Cartesian bicategories I"
- Walters, 2009, blog post, "Categorical algebras of relations",
  http://rfcwalters.blogspot.com/2009/10/categorical-algebras-of-relations.html
"""
@signature BicategoryRelations{Ob,Hom} <: HypergraphCategory{Ob,Hom} begin
  # Logical operations.
  meet(R::(A → B), S::(A → B))::(A → B) ⊣ (A::Ob, B::Ob)
  top(A::Ob, B::Ob)::(A → B)
end

@syntax FreeBicategoryRelations{ObExpr,HomExpr} BicategoryRelations begin
  otimes(A::Ob, B::Ob) = associate_unit(new(A,B), munit)
  otimes(R::Hom, S::Hom) = associate(new(R,S))
  compose(R::Hom, S::Hom) = associate(new(R,S; strict=true))
  dagger(R::Hom) = distribute_unary(distribute_dagger(involute(new(R))),
                                    dagger, otimes)
  meet(R::Hom, S::Hom) = compose(mcopy(dom(R)), otimes(R,S), mmerge(codom(R)))
  top(A::Ob, B::Ob) = compose(delete(A), create(B))
end

""" Theory of *abelian bicategories of relations*

Unlike [`BicategoryRelations`](@ref), this theory uses additive notation.

References:

- Carboni & Walters, 1987, "Cartesian bicategories I", Sec. 5
- Baez & Erbele, 2015, "Categories in control"
"""
@signature AbelianBicategoryRelations{Ob,Hom} <: HypergraphCategoryAdditive{Ob,Hom} begin
  # Second supply of Frobenius monoids.
  plus(A::Ob)::((A ⊕ A) → A)
  zero(A::Ob)::(mzero() → A)
  coplus(A::Ob)::(A → (A ⊕ A))
  cozero(A::Ob)::(A → mzero())

  # Logical operations.
  meet(R::(A → B), S::(A → B))::(A → B) ⊣ (A::Ob, B::Ob)
  top(A::Ob, B::Ob)::(A → B)
  join(R::(A → B), S::(A → B))::(A → B) ⊣ (A::Ob, B::Ob)
  bottom(A::Ob, B::Ob)::(A → B)
end

@syntax FreeAbelianBicategoryRelations{ObExpr,HomExpr} AbelianBicategoryRelations begin
  oplus(A::Ob, B::Ob) = associate_unit(new(A,B), mzero)
  oplus(R::Hom, S::Hom) = associate(new(R,S))
  compose(R::Hom, S::Hom) = associate(new(R,S; strict=true))
  dagger(R::Hom) = distribute_unary(distribute_dagger(involute(new(R))),
                                    dagger, oplus)
  meet(R::Hom, S::Hom) = compose(mcopy(dom(R)), oplus(R,S), mmerge(codom(R)))
  join(R::Hom, S::Hom) = compose(coplus(dom(R)), oplus(R,S), plus(codom(R)))
  top(A::Ob, B::Ob) = compose(delete(A), create(B))
  bottom(A::Ob, B::Ob) = compose(cozero(A), zero(B))
end

""" Theory of a *distributive bicategory of relations*

References:
  - Carboni & Walters, 1987, "Cartesian bicategories I", Remark 3.7 (mention
    in passing only)
  - Patterson, 2017, "Knowledge representation in bicategories of relations",
    Section 9.2

FIXME: Should also inherit `BicategoryOfRelations`, but multiple inheritance is
not yet supported.
"""
@signature DistributiveBicategoryRelations{Ob,Hom} <:
    DistributiveMonoidalCategoryWithDiagonals{Ob,Hom} begin
  # Self-dual dagger compact category.
  dagger(R::(A → B))::(B → A) ⊣ (A::Ob, B::Ob)
  dunit(A::Ob)::(munit() → (A ⊗ A))
  dcounit(A::Ob)::((A ⊗ A) → munit())

  # Merging and creating (right adjoints of copying and deleting maps).
  mmerge(A::Ob)::((A ⊗ A) → A)
  @op (∇) := mmerge
  create(A::Ob)::(munit() → A)
  @op (□) := create

  # Co-addition and co-zero (right adjoints of addition and zero maps).
  coplus(A::Ob)::(A → (A ⊕ A))
  cozero(A::Ob)::(A → mzero())

  # The coproduct is automatically a biproduct, due to compact closed structure.
  pair(R::(A → B), S::(A → C))::(A → (B ⊕ C)) ⊣ (A::Ob, B::Ob, C::Ob)
  proj1(A::Ob, B::Ob)::((A ⊕ B) → A)
  proj2(A::Ob, B::Ob)::((A ⊕ B) → B)

  # Logical operations.
  meet(R::(A → B), S::(A → B))::(A → B) ⊣ (A::Ob, B::Ob)
  top(A::Ob, B::Ob)::(A → B)
  join(R::(A → B), S::(A → B))::(A → B) ⊣ (A::Ob, B::Ob)
  bottom(A::Ob, B::Ob)::(A → B)
end


text:
@safetestset "isjwt" begin
    using Validator

    @test Validator.isjwt("eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJsb2dnZWRJbkFzIjoiYWRtaW4iLCJpYXQiOjE0MjI3Nzk2Mzh9.gzSraSYS8EXBxLN_oWnFSRgCzcmJmMjLiuyu5CSpyHI") === true
    @test Validator.isjwt("eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJsb3JlbSI6Imlwc3VtIn0.ymiJSsMJXR6tMSr8G9usjQ15_8hKPDv_CArLhxw28MI") === true
    @test Validator.isjwt("eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJkb2xvciI6InNpdCIsImFtZXQiOlsibG9yZW0iLCJpcHN1bSJdfQ.rRpe04zbWbbJjwM43VnHzAboDzszJtGrNsUxaqQ-GQ8") === true
    @test Validator.isjwt("eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJqb2huIjp7ImFnZSI6MjUsImhlaWdodCI6MTg1fSwiamFrZSI6eyJhZ2UiOjMwLCJoZWlnaHQiOjI3MH19.YRLPARDmhGMC3BBk_OhtwwK21PIkVCqQe8ncIRPKo-E") === true
    @test Validator.isjwt("eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ") === true

    @test Validator.isjwt("eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9") === false
    @test Validator.isjwt("\$Zs.ewu.su84") === false
    @test Validator.isjwt("ks64\$S/9.dy\$§kz.3sd73b") === false
end


text:
struct PkgEntry
    path::String
    registry_path::String
    name::String
    uuid::UUID
    repo::String
end


text:
@testset "Frobenius-Schur & projections" begin
    G = SmallPermGroups[8][4]
    irr = Characters.irreducible_characters(G)

    # quaternionic character
    χ = irr[1]
    @test collect(values(χ)) == [2, 0, 0, -2, 0]

    ι = Characters.frobenius_schur

    @test ι(χ) == -1

    @test deepcopy(χ) == χ
    @test deepcopy(χ) !== χ

    ψ = deepcopy(χ)

    @test ψ == χ
    @test ψ !== χ
    @test ψ == deepcopy(χ)

    @test Characters.conjugacy_classes(ψ) === Characters.conjugacy_classes(χ)
    @test Characters.table(ψ) === Characters.table(χ)

    @test collect(values(ψ)) == collect(values(χ))
    @test Characters.constituents(ψ) == Characters.constituents(χ)
    @test Characters.constituents(ψ) !== Characters.constituents(χ)

    @test hash(ψ) == hash(χ)

    @test 2ψ == 2χ
    @test 2ψ/2 == χ
    @test 2ψ != χ

    @test PermutationGroups.degree(χ) == 2

    # @test size(SymbolicWedderburn.image_basis(χ), 1) == 4

    @test_throws AssertionError ι(2χ)
    a = 2χ
    @test sum(length(cc)*a(first(cc)^2) for cc in Characters.conjugacy_classes(a))//order(G) == 2ι(χ)
end

@testset "Characters io" begin
    G = PermGroup([perm"(2,3)(4,5)"])
    chars = Characters.irreducible_characters(G)

    @test sprint(show, chars[1]) == "χ₁"
    @test sprint(show, chars[2]) == "χ₂"
    @test sprint(show, chars[1]+2chars[2]) == "χ₁ +2·χ₂"
    @test sprint(show, 2chars[1]-chars[2]) == "2·χ₁ -1·χ₂"
    @test sprint(show, -2chars[1]+3chars[2]) == "-2·χ₁ +3·χ₂"

    if VERSION > v"1.3.0"
        @test sprint(show, MIME"text/plain"(), chars[1]) ==
        "Character over Cyclotomic{Rational{Int64}, SparseArrays.SparseVector{Rational{Int64}, Int64}}\nχ₁"
    end
end


text:
#=Copyright (c) 2017 Gabriel Goh

Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
the Software, and to permit persons to whom the Software is furnished to do so,
subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.=#

using ConicIP

set_default_solver(ConicIPSolver(optTol = 1e-10))

b = 0.5*ones(10)

function Ψ(λ)
  x = Variable(15)
  p = minimize( 0.5*sumsquares(A*x - b) + λ*norm(y,1) )
  solve!(p)
  s = y.value
  s[abs(s) .< 1e-8] = 0
  return s
end

x = 0.5*ones(10)
W = randn(10,15)
I = eye(10)
ϵ = 0.001

K = !(Ψ(x) .== 0)
ystar = Ψ(x)
inv(W[:,K]'W[:,K])*(W[:,K]'x - sign(ystar[K]))
Ψ(x)

(Ψ(x + ϵ*I[:,1]) - Ψ(x - ϵ*I[:,1]))/(2ϵ)

inv(W[:,K]'W[:,K])*(W[:,K]')

max_stars_repo_name: sbeleidy/post--momentum

max_issues_repo_name: sbeleidy/post--momentum

max_forks_repo_name: sbeleidy/post--momentum


text:
"Variable Clustering Model Hyperparameters"
immutable VCMHyperparam <: AbstractHyperparam

  μ_x::Float64
  σ_x::Float64

  μ_g::Float64
  σ_g::Float64

  μ_noise::Float64
  σ_noise::Float64

  γ_noise_a::Float64
  γ_noise_b::Float64

  γ_g_a::Float64
  γ_g_b::Float64

  γ_alpha_a::Float64
  γ_alpha_b::Float64

  VCMHyperparam() = new(0.0, 1.0, 0.0, 1.0, 0.0, 0.1, 1.0, 0.1, 1.0, 1.0, 1.0, 1.0)

end

"Variable Clustering Model Data Object"
type VCMData <: AbstractModelData

    # energy
    energy::Float64

    # latent factor matrix
    X::Array{Float64}

    # latent factor loadings
    G::Array{Float64}

    # assignment Matrix
    C::SparseMatrixCSC

    # number of clusters
    K::Int

    # concentration parameter (Dirichlet)
    α::Float64

    # sigma of Gaussian noise
    σ_noise::Float64

    # sigma of Gaussian prior on factor loadings
    σ_g::Float64

end

type VCMBuffer <: AbstractModelBuffer

  μ_x::Float64
  σ_x::Float64

  μ_g::Float64
  σ_g::Float64

  μ_noise::Float64
  σ_noise::Float64

  # Data Set
  Y::Array{Float64}

  # Dimesnionality
  D::Int

  # Number of Samples
  N::Int

  # latent factor matrix
  X::Array{Float64}

  # latent factor loadings
  G::Array{Float64}

  # assignment Matrix
  C::SparseMatrixCSC

  # assignments
  cc::Vector{Int}

  # number of clusters
  K::Int

  # concentration parameter (Dirichlet)
  α::Float64

  # number of aux. variables
  m_aux::Int

end

"Incremental initialisation as published by Palla et al."
function init_incremental_vcm(Y::Array{Float64}, hyper::VCMHyperparam; α = 1.0)

  (D, N) = size(Y)

  X = randn(1,N)
  G = randn(1,1)
  C = sparse(collect(1))
  cc = [1]

  B = VCMBuffer(
    hyper.μ_x,
    hyper.σ_x,
    hyper.μ_g,
    hyper.σ_g,
    hyper.μ_noise,
    hyper.σ_noise,
    Y,
    D,
    N,
    X,
    G,
    C,
    cc,
    1,
    α,
    3 )

  Yd = B.Y[1,:]

  sample_latent_factors!(Yd, B)
  B.Y = Yd
  gibbs_aux_assignment!(B)

  B.α = random_concentration_parameter(B.α, hyper.γ_alpha_a, hyper.γ_alpha_b, B.N, B.K)
  B.σ_noise = random_sigma_noise(B, hyper.γ_noise_a, hyper.γ_noise_b)
  B.σ_g = random_sigma_g(B, hyper.γ_g_a, hyper.γ_g_b)

  # loop over D-1 dimensions
  for d = 2:B.D

      Yd = cat(1, Yd, Y[d,:])

      # sample unobserved data from model
      B.Y = Yd
      gibbs_aux_assignment!(B, out_dim = d)
      # not sure why this is required...

      # run one iteration:
      sample_latent_factors!(Yd, B)
      gibbs_aux_assignment!(B)

      B.α = random_concentration_parameter(B.α, hyper.γ_alpha_a, hyper.γ_alpha_b, B.N, B.K)
      B.σ_noise = random_sigma_noise(B, hyper.γ_noise_a, hyper.γ_noise_b)
      B.σ_g = random_sigma_g(B, hyper.γ_g_a, hyper.γ_g_b)

  end

  B.Y = Y

  return B

end

"Train VCM using Gibbs, currently very slow."
function train_gibbs_vcm(B::VCMBuffer, hyper::VCMHyperparam; α = 1.0, burnin = 10, thinout = 1, maxiter = 100, verbose = false)

  results = VCMData[]

  B.α = α

  for i in 1:burnin

    sample_latent_factors!(B.Y, B)
    gibbs_aux_assignment!(B)

    B.α = random_concentration_parameter(B.α, hyper.γ_alpha_a, hyper.γ_alpha_b, B.N, B.K)
    B.σ_noise = random_sigma_noise(B, hyper.γ_noise_a, hyper.γ_noise_b)
    B.σ_g = random_sigma_g(B, hyper.γ_g_a, hyper.γ_g_b)
  end

  # run gibbs sweeps
  for i in 1:maxiter

    # run thinning
    for j in 1:thinout
        sample_latent_factors!(B.Y, B)
        gibbs_aux_assignment!(B)

        B.α = random_concentration_parameter(B.α, hyper.γ_alpha_a, hyper.γ_alpha_b, B.N, B.K)
        B.σ_noise = random_sigma_noise(B, hyper.γ_noise_a, hyper.γ_noise_b)
        B.σ_g = random_sigma_g(B, hyper.γ_g_a, hyper.γ_g_b)
    end

    # compute energy
    E = B.Y - ( (B.G .* B.C) * B.X )
    llh = -B.D*B.N*(log(B.σ_noise) + 0.5*log(2*pi)) - 0.5*sum(E.^2) / (B.σ_noise^2)

    # simplified normal pdf
    prior_g = -0.5 * (1.0/B.σ_g.^2) * sum( (B.G-repmat((B.μ_g * ones(B.D, 1)), 1, B.K)).^2 )
    prior_x = -0.5 * (1.0/B.σ_x.^2) * sum( (B.X-repmat((B.μ_x * ones(B.K, 1)), 1, B.N)).^2 )

    # dirichlet?
    Ns = map(c -> sum(B.cc .== c), unique(B.cc))
    prior_c = lgamma(B.α)+B.K*log(B.α) +sum(lgamma(Ns)) - lgamma(B.α+B.D)

    # energy
    e = llh + prior_g + prior_x + prior_c

    # store result of current Gibbs sweep
    push!(results, VCMData(e, deepcopy(B.X), deepcopy(B.G), deepcopy(B.C), B.K, B.α, B.σ_noise, B.σ_g))

    if verbose
      println("iteration: ", i, " with energy: ", e, " K: ", K)
    end
  end

  return results

end


text:
function priorprob(parameters::Array{Float64, 1}, prior::Prior)
  pprob = 1
  for i in 1:length(parameters)
      pprob = pprob * pdf(prior.distribution[i], parameters[i])
  end
  return pprob
end

function kernelprob(p1, p2, kernel::Kernel)
    prob = 1
    for i in 1:length(p1.params)
      prob = prob * kernel.pdf_function(p1, p2, kernel.kernel_parameters, i)
    end
    return prob
end

function getmodelfreq(particles, ABCsetup)
  freq = zeros(Int64, ABCsetup.nmodels)
  models = map(x -> x.model, particles)
  for i in 1:ABCsetup.nmodels
    freq[i] = sum(models.==i)
  end
  return freq
end

function getmodelprob(currmodel, prevmodel, modelprob, ABCsetup)
  prob = ABCsetup.modelkern
  if currmodel == prevmodel
    return prob
  elseif sum(modelprob.>0.0) > 1
    return (1 - prob) / (sum(modelprob.>0.0) - 1)
  else
    return prob
  end
end

function smcweights(particles, oldparticles, prior, kernel::Kernel)

  weights = zeros(Float64, length(particles))

  for i in 1:length(particles)
    numerator = priorprob(particles[i].params, prior)
    denominator = 0.0
    for j in 1:length(particles)
      denominator = denominator + oldparticles[j].weight * kernelprob(particles[i], oldparticles[j], kernel)
    end
    weights[i] = numerator / denominator
  end

  weights = weights ./ sum(weights)
  for i in 1:length(particles)
    particles[i].weight = weights[i]
  end

  return particles, weights
end

#the below functions are related to the SMC with model selection algorithm

function modelperturbation(p, prevmodelprob, ABCsetup)
  denominator_m = 0.0
  for i in 1:ABCsetup.nmodels
    denominator_m = denominator_m + prevmodelprob[i] * getmodelprob(p.model, i, prevmodelprob, ABCsetup)
  end
  return denominator_m
end

function parameterperturbation(p, particles, prevmodelprob, kernel)
  denominator_p = 0.0
  for i in particles
    if p.model == i.model
      denominator_p = denominator_p + ((i.weight * kernelprob(p, i, kernel)) / prevmodelprob[p.model])
    end
  end
  return denominator_p
end

function smcweightsmodel(particles, oldparticles, ABCsetup, prevmodelprob)

  #calculate weights of each particle
  numerator = zeros(Float64, length(particles))
  modeldenominator = zeros(Float64, length(particles))
  particledenominator = zeros(Float64, length(particles))

  for i in 1:length(particles)
    # get prior probabilty of parameter set
    numerator[i] = priorprob(particles[i].params, ABCsetup.Models[particles[i].model].prior)
    # get probability of parameter perturbation
    particledenominator[i] = parameterperturbation(particles[i], oldparticles, prevmodelprob, ABCsetup.Models[particles[i].model].kernel)
    # get probability of model perturbation
    modeldenominator[i] = modelperturbation(particles[i], prevmodelprob, ABCsetup)
  end

  weights = numerator ./ (modeldenominator .* particledenominator)
  weights = weights ./ sum(weights)

  for i in 1:length(particles)
    particles[i].weight = weights[i]
  end

  return particles, weights
end

function getparticleweights(particles, ABCsetup)
  w = zeros(Float64, ABCsetup.nmodels, ABCsetup.nparticles)
  for i in 1:ABCsetup.nparticles
    w[particles[i].model, i] = particles[i].weight
  end
  weights = w ./ sum(w, dims = 2)
  return weights, sum(w, dims = 2)[:]
end


text:
using MetaICVI
# using ScikitLearn: fit!, score, @sk_import
# using ScikitLearn: fit!, score
using ScikitLearn.CrossValidation: train_test_split
using ScikitLearn
# using ProgressMeter
using PyCallJLD
# @sk_import linear_model:RidgeClassifier
# learner = RidgeClassifier()

# Include local utils
# include("../test/test_utils.jl")

# Identify the rocket file
# rocket_file = "data/models/rocket.jld2"
# classifier_file = "data/models/classifier.jld"

# # Point to data
# data_dir(args...) = joinpath("data/training", args...)
data_dir = "data/training"

# Create the options
opts = MetaICVIOpts(
    # rocket_file = rocket_file,
    # classifier_file = classifier_file,
    n_rocket = 20
)

# Create the metaicvi object
metaicvi = MetaICVIModule(opts)

(features_data, features_targets) = get_training_features(metaicvi, data_dir)

# Create a split for training/testing the learner, correct for how scikitlearn expects features
x_train, x_test, y_train, y_test = train_test_split(transpose(features_data), features_targets, test_size=0.4)

# Train and test the learner
# fit!(metaicvi.classifier, x_train, y_train)
train_and_save(metaicvi, x_train, y_train)
performance = score(metaicvi.classifier, x_test, y_test)

@info "Performance is: $performance"


text:
mutable struct State{F <: AbstractFloat, V <: AbstractVector{<:Real}}
  p::F
  beta::Bool
  gammaC::F
  gammaT::F
  etaC::V  # K
  etaT::V  # K
  lambdaC::Vector{Int}  # NC
  lambdaT::Vector{Int}  # NT
  mu::V  # K
  nu::V  # K
  omega::V  # K. alt variance.
  psi::V  # K. alt skew.
  vC::V  # NC
  vT::V  # NT
  zetaC::V  # NC
  zetaT::V  # NT
  tau::F
  sigma::V
  phi::V
end

function State(data::Data, prior::Prior)
  p = rand(prior.p)
  beta = true
  gammaC = rand(prior.gamma)
  gammaT = rand(prior.gamma)
  etaC = rand(prior.eta)
  etaT = rand(prior.eta)
  lambdaC = rand(Categorical(etaC), data.NC_finite)
  lambdaT = rand(Categorical(etaT), data.NT_finite)
  mu = rand(prior.mu, prior.K)
  nu = rand(prior.nu, prior.K)
  tau = rand(prior.tau)
  prior_omega = InverseGamma(prior.a_omega, tau)
  omega = rand(prior_omega, prior.K)
  psi = rand(prior.psi, prior.K)
  vC = [rand(Gamma(nu[k]/2, 2/nu[k])) for k in lambdaC]
  vT = [rand(Gamma(nu[k]/2, 2/nu[k])) for k in lambdaT]
  zetaC = [rand(truncated(Normal(0, sqrt(1/v)), 0, Inf)) for v in vC]
  zetaT = [rand(truncated(Normal(0, sqrt(1/v)), 0, Inf)) for v in vT]
  sigma = Util.scalefromaltskewt.(sqrt.(omega), psi)
  phi = Util.skewfromaltskewt.(sqrt.(omega), psi)

  state = State(p, beta, gammaC, gammaT, etaC, etaT, lambdaC, lambdaT,
                mu, nu, omega, psi, vC, vT, zetaC, zetaT, tau, sigma, phi)

  return state
end

ref_lambda(state::State, i::Char) = (i == 'C') ? state.lambdaC : state.lambdaT
ref_v(state::State, i::Char) = (i == 'C') ? state.vC : state.vT
ref_zeta(state::State, i::Char) = (i == 'C') ? state.zetaC : state.zetaT

ref_eta(state::State, i::Char) = (i == 'C') ? state.etaC : state.etaT
ref_gamma(state::State, i::Char) = (i == 'C') ? state.gammaC : state.gammaT

max_stars_repo_name: luiarthur/CytofDensityEstimation

max_issues_repo_name: luiarthur/CytofDensityEstimation

max_forks_repo_name: luiarthur/CytofDensityEstimation


text:
s = open("input.txt") do file
    readlines(file)
end

function GetID(seatCode)
    charArr = seatCode |> collect;
    binString = String(map(x->ifelse(x=='F','0','1'),charArr[1:7]));

    row = parse(Int, binString, base=2);

    binString = String(map(x->ifelse(x=='L','0','1'),charArr[8:10]));
    col = parse(Int, binString, base=2);

    return row * 8 + col;
end

maximum(map(GetID, s))