text:
"""
    AbstractDataFrame

An abstract type for which all concrete types expose an interface
for working with tabular data.

# Common methods

An `AbstractDataFrame` is a two-dimensional table with `Symbol`s or strings
for column names.

The following are normally implemented for AbstractDataFrames:

* [`describe`](@ref) : summarize columns
* `summary` : show number of rows and columns
* `hcat` : horizontal concatenation
* `vcat` : vertical concatenation
* [`repeat`](@ref) : repeat rows
* `names` : columns names
* [`rename!`](@ref) : rename columns names based on keyword arguments
* `length` : number of columns
* `size` : (nrows, ncols)
* [`first`](@ref) : first `n` rows
* [`last`](@ref) : last `n` rows
* `convert` : convert to an array
* [`completecases`](@ref) : boolean vector of complete cases (rows with no missings)
* [`dropmissing`](@ref) : remove rows with missing values
* [`dropmissing!`](@ref) : remove rows with missing values in-place
* [`nonunique`](@ref) : indexes of duplicate rows
* [`unique`](@ref) : remove duplicate rows
* [`unique!`](@ref) : remove duplicate rows in-place
* [`disallowmissing`](@ref) : drop support for missing values in columns
* [`disallowmissing!`](@ref) : drop support for missing values in columns in-place
* [`allowmissing`](@ref) : add support for missing values in columns
* [`allowmissing!`](@ref) : add support for missing values in columns in-place
* [`categorical`](@ref) : change column types to categorical
* [`categorical!`](@ref) : change column types to categorical in-place
* `similar` : a DataFrame with similar columns as `d`
* `filter` : remove rows
* `filter!` : remove rows in-place

# Indexing and broadcasting

`AbstractDataFrame` can be indexed by passing two indices specifying
row and column selectors. The allowed indices are a superset of indices
that can be used for standard arrays. You can also access a single column
of an `AbstractDataFrame` using `getproperty` and `setproperty!` functions.
Columns can be selected using integers, `Symbol`s, or strings.
In broadcasting `AbstractDataFrame` behavior is similar to a `Matrix`.

A detailed description of `getindex`, `setindex!`, `getproperty`, `setproperty!`,
broadcasting and broadcasting assignment for data frames is given in
the ["Indexing" section](https://juliadata.github.io/DataFrames.jl/stable/lib/indexing/)
of the manual.

"""
abstract type AbstractDataFrame end

##############################################################################
##
## Basic properties of a DataFrame
##
##############################################################################

"""
    names(df::AbstractDataFrame)
    names(df::AbstractDataFrame, cols)

Return a freshly allocated `Vector{String}` of names of columns contained in `df`.

If `cols` is passed then restrict returned column names to those matching the
selector (this is useful in particular with regular expressions, `Not`, and `Between`).
`cols` can be any column selector ($COLUMNINDEX_STR; $MULTICOLUMNINDEX_STR).

See also [`propertynames`](@ref) which returns a `Vector{Symbol}`.
"""
Base.names(df::AbstractDataFrame) = names(index(df))

function Base.names(df::AbstractDataFrame, cols)
    nms = _names(index(df))
    idx = index(df)[cols]
    idxs = idx isa Int ? (idx:idx) : idx
    return [string(nms[i]) for i in idxs]
end

# _names returns Vector{Symbol} without copying
_names(df::AbstractDataFrame) = _names(index(df))

# separate methods are needed due to dispatch ambiguity
Compat.hasproperty(df::AbstractDataFrame, s::Symbol) = haskey(index(df), s)
Compat.hasproperty(df::AbstractDataFrame, s::AbstractString) = haskey(index(df), s)

"""
    rename!(df::AbstractDataFrame, vals::AbstractVector{Symbol};
            makeunique::Bool=false)
    rename!(df::AbstractDataFrame, vals::AbstractVector{<:AbstractString};
            makeunique::Bool=false)
    rename!(df::AbstractDataFrame, (from => to)::Pair...)
    rename!(df::AbstractDataFrame, d::AbstractDict)
    rename!(df::AbstractDataFrame, d::AbstractVector{<:Pair})
    rename!(f::Function, df::AbstractDataFrame)

Rename columns of `df` in-place.
Each name is changed at most once. Permutation of names is allowed.

# Arguments
- `df` : the `AbstractDataFrame`
- `d` : an `AbstractDict` or an `AbstractVector` of `Pair`s that maps
  the original names or column numbers to new names
- `f` : a function which for each column takes the old name as a `String`
  and returns the new name that gets converted to a `Symbol`
- `vals` : new column names as a vector of `Symbol`s or `AbstractString`s
  of the same length as the number of columns in `df`
- `makeunique` : if `false` (the default), an error will be raised
  if duplicate names are found; if `true`, duplicate names will be suffixed
  with `_i` (`i` starting at 1 for the first duplicate).

If pairs are passed to `rename!` (as positional arguments or in a dictionary or
a vector) then:
* `from` value can be a `Symbol`, an `AbstractString` or an `Integer`;
* `to` value can be a `Symbol` or an `AbstractString`.

Mixing symbols and strings in `to` and `from` is not allowed.

See also: [`rename`](@ref)

# Examples
```julia
julia> df = DataFrame(i = 1, x = 2, y = 3)
1×3 DataFrame
│ Row │ i     │ x     │ y     │
│     │ Int64 │ Int64 │ Int64 │
├─────┼───────┼───────┼───────┤
│ 1   │ 1     │ 2     │ 3     │

julia> rename!(df, Dict(:i => "A", :x => "X"))
1×3 DataFrame
│ Row │ A     │ X     │ y     │
│     │ Int64 │ Int64 │ Int64 │
├─────┼───────┼───────┼───────┤
│ 1   │ 1     │ 2     │ 3     │

julia> rename!(df, [:a, :b, :c])
1×3 DataFrame
│ Row │ a     │ b     │ c     │
│     │ Int64 │ Int64 │ Int64 │
├─────┼───────┼───────┼───────┤
│ 1   │ 1     │ 2     │ 3     │

julia> rename!(df, [:a, :b, :a])
ERROR: ArgumentError: Duplicate variable names: :a. Pass makeunique=true to make
them unique using a suffix automatically.

julia> rename!(df, [:a, :b, :a], makeunique=true)
1×3 DataFrame
│ Row │ a     │ b     │ a_1   │
│     │ Int64 │ Int64 │ Int64 │
├─────┼───────┼───────┼───────┤
│ 1   │ 1     │ 2     │ 3     │

julia> rename!(uppercase, df)
1×3 DataFrame
│ Row │ A     │ B     │ A_1   │
│     │ Int64 │ Int64 │ Int64 │
├─────┼───────┼───────┼───────┤
│ 1   │ 1     │ 2     │ 3     │
```
"""
function rename!(df::AbstractDataFrame, vals::AbstractVector{Symbol};
                 makeunique::Bool=false)
    rename!(index(df), vals, makeunique=makeunique)
    return df
end

function rename!(df::AbstractDataFrame, vals::AbstractVector{<:AbstractString};
                 makeunique::Bool=false)
    rename!(index(df), Symbol.(vals), makeunique=makeunique)
    return df
end

function rename!(df::AbstractDataFrame, args::AbstractVector{Pair{Symbol,Symbol}})
    rename!(index(df), args)
    return df
end

function rename!(df::AbstractDataFrame,
                 args::Union{AbstractVector{<:Pair{Symbol,<:AbstractString}},
                             AbstractVector{<:Pair{<:AbstractString,Symbol}},
                             AbstractVector{<:Pair{<:AbstractString,<:AbstractString}},
                             AbstractDict{Symbol,Symbol},
                             AbstractDict{Symbol,<:AbstractString},
                             AbstractDict{<:AbstractString,Symbol},
                             AbstractDict{<:AbstractString,<:AbstractString}})
    rename!(index(df), [Symbol(from) => Symbol(to) for (from, to) in args])
    return df
end

function rename!(df::AbstractDataFrame,
                 args::Union{AbstractVector{<:Pair{<:Integer,<:AbstractString}},
                             AbstractVector{<:Pair{<:Integer,Symbol}},
                             AbstractDict{<:Integer,<:AbstractString},
                             AbstractDict{<:Integer,Symbol}})
    rename!(index(df), [_names(df)[from] => Symbol(to) for (from, to) in args])
    return df
end

rename!(df::AbstractDataFrame, args::Pair...) = rename!(df, collect(args))

function rename!(f::Function, df::AbstractDataFrame)
    rename!(f, index(df))
    return df
end

"""
    rename(df::AbstractDataFrame, vals::AbstractVector{Symbol};
           makeunique::Bool=false)
    rename(df::AbstractDataFrame, vals::AbstractVector{<:AbstractString};
           makeunique::Bool=false)
    rename(df::AbstractDataFrame, (from => to)::Pair...)
    rename(df::AbstractDataFrame, d::AbstractDict)
    rename(df::AbstractDataFrame, d::AbstractVector{<:Pair})
    rename(f::Function, df::AbstractDataFrame)

Create a new data frame that is a copy of `df` with changed column names.
Each name is changed at most once. Permutation of names is allowed.

# Arguments
- `df` : the `AbstractDataFrame`; if it is a `SubDataFrame` then renaming is
  only allowed if it was created using `:` as a column selector.
- `d` : an `AbstractDict` or an `AbstractVector` of `Pair`s that maps
  the original names or column numbers to new names
- `f` : a function which for each column takes the old name as a `String`
  and returns the new name that gets converted to a `Symbol`
- `vals` : new column names as a vector of `Symbol`s or `AbstractString`s
  of the same length as the number of columns in `df`
- `makeunique` : if `false` (the default), an error will be raised
  if duplicate names are found; if `true`, duplicate names will be suffixed
  with `_i` (`i` starting at 1 for the first duplicate).

If pairs are passed to `rename` (as positional arguments or in a dictionary or
a vector) then:
* `from` value can be a `Symbol`, an `AbstractString` or an `Integer`;
* `to` value can be a `Symbol` or an `AbstractString`.

Mixing symbols and strings in `to` and `from` is not allowed.

See also: [`rename!`](@ref)

# Examples
```julia
julia> df = DataFrame(i = 1, x = 2, y = 3)
1×3 DataFrame
│ Row │ i     │ x     │ y     │
│     │ Int64 │ Int64 │ Int64 │
├─────┼───────┼───────┼───────┤
│ 1   │ 1     │ 2     │ 3     │

julia> rename(df, :i => :A, :x => :X)
1×3 DataFrame
│ Row │ A     │ X     │ y     │
│     │ Int64 │ Int64 │ Int64 │
├─────┼───────┼───────┼───────┤
│ 1   │ 1     │ 2     │ 3     │

julia> rename(df, :x => :y, :y => :x)
1×3 DataFrame
│ Row │ i     │ y     │ x     │
│     │ Int64 │ Int64 │ Int64 │
├─────┼───────┼───────┼───────┤
│ 1   │ 1     │ 2     │ 3     │

julia> rename(df, [1 => :A, 2 => :X])
1×3 DataFrame
│ Row │ A     │ X     │ y     │
│     │ Int64 │ Int64 │ Int64 │
├─────┼───────┼───────┼───────┤
│ 1   │ 1     │ 2     │ 3     │

julia> rename(df, Dict("i" => "A", "x" => "X"))
1×3 DataFrame
│ Row │ A     │ X     │ y     │
│     │ Int64 │ Int64 │ Int64 │
├─────┼───────┼───────┼───────┤
│ 1   │ 1     │ 2     │ 3     │

julia> rename(uppercase, df)
1×3 DataFrame
│ Row │ I     │ X     │ Y     │
│     │ Int64 │ Int64 │ Int64 │
├─────┼───────┼───────┼───────┤
│ 1   │ 1     │ 2     │ 3     │
```
"""
rename(df::AbstractDataFrame, vals::AbstractVector{Symbol};
       makeunique::Bool=false) = rename!(copy(df), vals, makeunique=makeunique)
rename(df::AbstractDataFrame, vals::AbstractVector{<:AbstractString};
       makeunique::Bool=false) = rename!(copy(df), vals, makeunique=makeunique)
rename(df::AbstractDataFrame, args...) = rename!(copy(df), args...)
rename(f::Function, df::AbstractDataFrame) = rename!(f, copy(df))

"""
    size(df::AbstractDataFrame, [dim])

Return a tuple containing the number of rows and columns of `df`.
Optionally a dimension `dim` can be specified, where `1` corresponds to rows
and `2` corresponds to columns.

See also: [`nrow`](@ref), [`ncol`](@ref)

# Examples
```julia
julia> df = DataFrame(a=1:3, b='a':'c');

julia> size(df)
(3, 2)

julia> size(df, 1)
3
```
"""
Base.size(df::AbstractDataFrame) = (nrow(df), ncol(df))
function Base.size(df::AbstractDataFrame, i::Integer)
    if i == 1
        nrow(df)
    elseif i == 2
        ncol(df)
    else
        throw(ArgumentError("DataFrames only have two dimensions"))
    end
end

Base.isempty(df::AbstractDataFrame) = size(df, 1) == 0 || size(df, 2) == 0

Base.lastindex(df::AbstractDataFrame, i::Integer) = last(axes(df, i))
Base.axes(df::AbstractDataFrame, i::Integer) = Base.OneTo(size(df, i))

"""
    ndims(::AbstractDataFrame)
    ndims(::Type{<:AbstractDataFrame})

Return the number of dimensions of a data frame, which is always `2`.
"""
Base.ndims(::AbstractDataFrame) = 2
Base.ndims(::Type{<:AbstractDataFrame}) = 2

# separate methods are needed due to dispatch ambiguity
Base.getproperty(df::AbstractDataFrame, col_ind::Symbol) = df[!, col_ind]
Base.getproperty(df::AbstractDataFrame, col_ind::AbstractString) = df[!, col_ind]

# Private fields are never exposed since they can conflict with column names
"""
    propertynames(df::AbstractDataFrame)

Return a freshly allocated `Vector{Symbol}` of names of columns contained in `df`.
"""
Base.propertynames(df::AbstractDataFrame, private::Bool=false) = copy(_names(df))

##############################################################################
##
## Similar
##
##############################################################################

"""
    similar(df::AbstractDataFrame, rows::Integer=nrow(df))

Create a new `DataFrame` with the same column names and column element types
as `df`. An optional second argument can be provided to request a number of rows
that is different than the number of rows present in `df`.
"""
function Base.similar(df::AbstractDataFrame, rows::Integer = size(df, 1))
    rows < 0 && throw(ArgumentError("the number of rows must be non-negative"))
    DataFrame(AbstractVector[similar(x, rows) for x in eachcol(df)], copy(index(df)),
              copycols=false)
end

"""
    empty(df::AbstractDataFrame)

Create a new `DataFrame` with the same column names and column element types
as `df` but with zero rows.
"""
Base.empty(df::AbstractDataFrame) = similar(df, 0)

##############################################################################
##
## Equality
##
##############################################################################

function Base.:(==)(df1::AbstractDataFrame, df2::AbstractDataFrame)
    size(df1, 2) == size(df2, 2) || return false
    isequal(index(df1), index(df2)) || return false
    eq = true
    for idx in 1:size(df1, 2)
        coleq = df1[!, idx] == df2[!, idx]
        # coleq could be missing
        isequal(coleq, false) && return false
        eq &= coleq
    end
    return eq
end

function Base.isequal(df1::AbstractDataFrame, df2::AbstractDataFrame)
    size(df1, 2) == size(df2, 2) || return false
    isequal(index(df1), index(df2)) || return false
    for idx in 1:size(df1, 2)
        isequal(df1[!, idx], df2[!, idx]) || return false
    end
    return true
end

##############################################################################
##
## Description
##
##############################################################################

"""
    first(df::AbstractDataFrame)

Get the first row of `df` as a `DataFrameRow`.
"""
Base.first(df::AbstractDataFrame) = df[1, :]

"""
    first(df::AbstractDataFrame, n::Integer)

Get a data frame with the `n` first rows of `df`.
"""
Base.first(df::AbstractDataFrame, n::Integer) = df[1:min(n,nrow(df)), :]

"""
    last(df::AbstractDataFrame)

Get the last row of `df` as a `DataFrameRow`.
"""
Base.last(df::AbstractDataFrame) = df[nrow(df), :]

"""
    last(df::AbstractDataFrame, n::Integer)

Get a data frame with the `n` last rows of `df`.
"""
Base.last(df::AbstractDataFrame, n::Integer) = df[max(1,nrow(df)-n+1):nrow(df), :]

"""
    describe(df::AbstractDataFrame; cols=:)
    describe(df::AbstractDataFrame, stats::Union{Symbol, Pair}...; cols=:)

Return descriptive statistics for a data frame as a new `DataFrame`
where each row represents a variable and each column a summary statistic.

# Arguments
- `df` : the `AbstractDataFrame`
- `stats::Union{Symbol, Pair}...` : the summary statistics to report.
  Arguments can be:
    - A symbol from the list `:mean`, `:std`, `:min`, `:q25`,
      `:median`, `:q75`, `:max`, `:eltype`, `:nunique`, `:first`, `:last`, and
      `:nmissing`. The default statistics used are `:mean`, `:min`, `:median`,
      `:max`, `:nmissing`, and `:eltype`.
    - `:all` as the only `Symbol` argument to return all statistics.
    - A `name => function` pair where `name` is a `Symbol` or string. This will
      create a column of summary statistics with the provided name.
- `cols` : a keyword argument allowing to select only a subset of columns from `df`
  to describe. Can be any column selector ($COLUMNINDEX_STR; $MULTICOLUMNINDEX_STR).

# Details
For `Real` columns, compute the mean, standard deviation, minimum, first
quantile, median, third quantile, and maximum. If a column does not derive from
`Real`, `describe` will attempt to calculate all statistics, using `nothing` as
a fall-back in the case of an error.

When `stats` contains `:nunique`, `describe` will report the
number of unique values in a column. If a column's base type derives from `Real`,
`:nunique` will return `nothing`s.

Missing values are filtered in the calculation of all statistics, however the
column `:nmissing` will report the number of missing values of that variable.

If custom functions are provided, they are called repeatedly with the vector
corresponding to each column as the only argument. For columns allowing for
missing values, the vector is wrapped in a call to `skipmissing`: custom
functions must therefore support such objects (and not only vectors), and cannot
access missing values.

# Examples
```julia
julia> df = DataFrame(i=1:10, x=0.1:0.1:1.0, y='a':'j');

julia> describe(df)
3×7 DataFrame
│ Row │ variable │ mean   │ min │ median │ max │ nmissing │ eltype   │
│     │ Symbol   │ Union… │ Any │ Union… │ Any │ Int64    │ DataType │
├─────┼──────────┼────────┼─────┼────────┼─────┼──────────┼──────────┤
│ 1   │ i        │ 5.5    │ 1   │ 5.5    │ 10  │ 0        │ Int64    │
│ 2   │ x        │ 0.55   │ 0.1 │ 0.55   │ 1.0 │ 0        │ Float64  │
│ 3   │ y        │        │ 'a' │        │ 'j' │ 0        │ Char     │

julia> describe(df, :min, :max)
3×3 DataFrame
│ Row │ variable │ min │ max │
│     │ Symbol   │ Any │ Any │
├─────┼──────────┼─────┼─────┤
│ 1   │ i        │ 1   │ 10  │
│ 2   │ x        │ 0.1 │ 1.0 │
│ 3   │ y        │ 'a' │ 'j' │

julia> describe(df, :min, :sum => sum)
3×3 DataFrame
│ Row │ variable │ min │ sum │
│     │ Symbol   │ Any │ Any │
├─────┼──────────┼─────┼─────┤
│ 1   │ i        │ 1   │ 55  │
│ 2   │ x        │ 0.1 │ 5.5 │
│ 3   │ y        │ 'a' │     │

julia> describe(df, :min, :sum => sum, cols=:x)
1×3 DataFrame
│ Row │ variable │ min     │ sum     │
│     │ Symbol   │ Float64 │ Float64 │
├─────┼──────────┼─────────┼─────────┤
│ 1   │ x        │ 0.1     │ 5.5     │
```
"""
DataAPI.describe(df::AbstractDataFrame,
                 stats::Union{Symbol, Pair{<:SymbolOrString}}...;
                 cols=:) =
    _describe(select(df, cols, copycols=false), collect(stats))

DataAPI.describe(df::AbstractDataFrame; cols=:) =
    _describe(select(df, cols, copycols=false),
              [:mean, :min, :median, :max, :nmissing, :eltype])

function _describe(df::AbstractDataFrame, stats::AbstractVector)
    predefined_funs = Symbol[s for s in stats if s isa Symbol]

    allowed_fields = [:mean, :std, :min, :q25, :median, :q75,
                      :max, :nunique, :nmissing, :first, :last, :eltype]

    if predefined_funs == [:all]
        predefined_funs = allowed_fields
        i = findfirst(s -> s == :all, stats)
        splice!(stats, i, allowed_fields) # insert in the stats vector to get a good order
    elseif :all in predefined_funs
        throw(ArgumentError("`:all` must be the only `Symbol` argument."))
    elseif !issubset(predefined_funs, allowed_fields)
        not_allowed = join(setdiff(predefined_funs, allowed_fields), ", :")
        allowed_msg = "\nAllowed fields are: :" * join(allowed_fields, ", :")
        throw(ArgumentError(":$not_allowed not allowed." * allowed_msg))
    end

    custom_funs = Pair[Symbol(s[1]) => s[2] for s in stats if s isa Pair]

    ordered_names = [s isa Symbol ? s : Symbol(first(s)) for s in stats]

    if !allunique(ordered_names)
        df_ord_names = DataFrame(ordered_names = ordered_names)
        duplicate_names = unique(ordered_names[nonunique(df_ord_names)])
        throw(ArgumentError("Duplicate names not allowed. Duplicated value(s) are: " *
                            ":$(join(duplicate_names, ", "))"))
    end

    # Put the summary stats into the return data frame
    data = DataFrame()
    data.variable = propertynames(df)

    # An array of Dicts for summary statistics
    col_stats_dicts = map(eachcol(df)) do col
        if eltype(col) >: Missing
            t = collect(skipmissing(col))
            d = get_stats(t, predefined_funs)
            get_stats!(d, t, custom_funs)
        else
            d = get_stats(col, predefined_funs)
            get_stats!(d, col, custom_funs)
        end

        if :nmissing in predefined_funs
            d[:nmissing] = count(ismissing, col)
        end

        if :first in predefined_funs
            d[:first] = isempty(col) ? nothing : first(col)
        end

        if :last in predefined_funs
            d[:last] = isempty(col) ? nothing : last(col)
        end

        if :eltype in predefined_funs
            d[:eltype] = eltype(col)
        end

        return d
    end

    for stat in ordered_names
        # for each statistic, loop through the columns array to find values
        # letting the comprehension choose the appropriate type
        data[!, stat] = [d[stat] for d in col_stats_dicts]
    end

    return data
end

# Compute summary statistics
# use a dict because we dont know which measures the user wants
# Outside of the `describe` function due to something with 0.7
function get_stats(col::AbstractVector, stats::AbstractVector{Symbol})
    d = Dict{Symbol, Any}()

    if :q25 in stats || :median in stats || :q75 in stats
        q = try quantile(col, [.25, .5, .75]) catch; (nothing, nothing, nothing) end
        d[:q25] = q[1]
        d[:median] = q[2]
        d[:q75] = q[3]
    end

    if :min in stats || :max in stats
        ex = try extrema(col) catch; (nothing, nothing) end
        d[:min] = ex[1]
        d[:max] = ex[2]
    end

    if :mean in stats || :std in stats
        m = try mean(col) catch end
        # we can add non-necessary things to d, because we choose what we need
        # in the main function
        d[:mean] = m

        if :std in stats
            d[:std] = try std(col, mean = m) catch end
        end
    end

    if :nunique in stats
        if eltype(col) <: Real
            d[:nunique] = nothing
        else
            d[:nunique] = try length(Set(col)) catch end
        end
    end

    return d
end

function get_stats!(d::Dict, col::AbstractVector, stats::AbstractVector{<:Pair})
    for stat in stats
        d[stat[1]] = try stat[2](col) catch end
    end
end

##############################################################################
##
## Miscellaneous
##
##############################################################################

"""
    completecases(df::AbstractDataFrame, cols=:)

Return a Boolean vector with `true` entries indicating rows without missing values
(complete cases) in data frame `df`.

If `cols` is provided, only missing values in the corresponding columns areconsidered.
`cols` can be any column selector ($COLUMNINDEX_STR; $MULTICOLUMNINDEX_STR).

See also: [`dropmissing`](@ref) and [`dropmissing!`](@ref).
Use `findall(completecases(df))` to get the indices of the rows.

# Examples

```julia
julia> df = DataFrame(i = 1:5,
                      x = [missing, 4, missing, 2, 1],
                      y = [missing, missing, "c", "d", "e"])
5×3 DataFrame
│ Row │ i     │ x       │ y       │
│     │ Int64 │ Int64?  │ String? │
├─────┼───────┼─────────┼─────────┤
│ 1   │ 1     │ missing │ missing │
│ 2   │ 2     │ 4       │ missing │
│ 3   │ 3     │ missing │ c       │
│ 4   │ 4     │ 2       │ d       │
│ 5   │ 5     │ 1       │ e       │

julia> completecases(df)
5-element BitArray{1}:
 false
 false
 false
  true
  true

julia> completecases(df, :x)
5-element BitArray{1}:
 false
  true
 false
  true
  true

julia> completecases(df, [:x, :y])
5-element BitArray{1}:
 false
 false
 false
  true
  true
```
"""
function completecases(df::AbstractDataFrame, col::Colon=:)
    if ncol(df) == 0
        throw(ArgumentError("Unable to compute complete cases of a " *
                            "data frame with no columns"))
    end
    res = trues(size(df, 1))
    for i in 1:size(df, 2)
        res .&= .!ismissing.(df[!, i])
    end
    res
end

completecases(df::AbstractDataFrame, col::ColumnIndex) =
    .!ismissing.(df[!, col])

completecases(df::AbstractDataFrame, cols::MultiColumnIndex) =
    completecases(df[!, cols])

"""
    dropmissing(df::AbstractDataFrame, cols=:; disallowmissing::Bool=true)

Return a copy of data frame `df` excluding rows with missing values.

If `cols` is provided, only missing values in the corresponding columns are considered.
`cols` can be any column selector ($COLUMNINDEX_STR; $MULTICOLUMNINDEX_STR).

If `disallowmissing` is `true` (the default) then columns specified in `cols` will
be converted so as not to allow for missing values using [`disallowmissing!`](@ref).

See also: [`completecases`](@ref) and [`dropmissing!`](@ref).

# Examples

```julia
julia> df = DataFrame(i = 1:5,
                      x = [missing, 4, missing, 2, 1],
                      y = [missing, missing, "c", "d", "e"])
5×3 DataFrame
│ Row │ i     │ x       │ y       │
│     │ Int64 │ Int64?  │ String? │
├─────┼───────┼─────────┼─────────┤
│ 1   │ 1     │ missing │ missing │
│ 2   │ 2     │ 4       │ missing │
│ 3   │ 3     │ missing │ c       │
│ 4   │ 4     │ 2       │ d       │
│ 5   │ 5     │ 1       │ e       │

julia> dropmissing(df)
2×3 DataFrame
│ Row │ i     │ x     │ y      │
│     │ Int64 │ Int64 │ String │
├─────┼───────┼───────┼────────┤
│ 1   │ 4     │ 2     │ d      │
│ 2   │ 5     │ 1     │ e      │

julia> dropmissing(df, disallowmissing=false)
2×3 DataFrame
│ Row │ i     │ x      │ y       │
│     │ Int64 │ Int64? │ String? │
├─────┼───────┼────────┼─────────┤
│ 1   │ 4     │ 2      │ d       │
│ 2   │ 5     │ 1      │ e       │

julia> dropmissing(df, :x)
3×3 DataFrame
│ Row │ i     │ x     │ y       │
│     │ Int64 │ Int64 │ String? │
├─────┼───────┼───────┼─────────┤
│ 1   │ 2     │ 4     │ missing │
│ 2   │ 4     │ 2     │ d       │
│ 3   │ 5     │ 1     │ e       │

julia> dropmissing(df, [:x, :y])
2×3 DataFrame
│ Row │ i     │ x     │ y      │
│     │ Int64 │ Int64 │ String │
├─────┼───────┼───────┼────────┤
│ 1   │ 4     │ 2     │ d      │
│ 2   │ 5     │ 1     │ e      │
```
"""
function dropmissing(df::AbstractDataFrame,
                     cols::Union{ColumnIndex, MultiColumnIndex}=:;
                     disallowmissing::Bool=true)
    newdf = df[completecases(df, cols), :]
    disallowmissing && disallowmissing!(newdf, cols)
    newdf
end

"""
    dropmissing!(df::AbstractDataFrame, cols=:; disallowmissing::Bool=true)

Remove rows with missing values from data frame `df` and return it.

If `cols` is provided, only missing values in the corresponding columns are considered.
`cols` can be any column selector ($COLUMNINDEX_STR; $MULTICOLUMNINDEX_STR).

If `disallowmissing` is `true` (the default) then the `cols` columns will
get converted using [`disallowmissing!`](@ref).

See also: [`dropmissing`](@ref) and [`completecases`](@ref).

```jldoctest
julia> df = DataFrame(i = 1:5,
                      x = [missing, 4, missing, 2, 1],
                      y = [missing, missing, "c", "d", "e"])
5×3 DataFrame
│ Row │ i     │ x       │ y       │
│     │ Int64 │ Int64?  │ String? │
├─────┼───────┼─────────┼─────────┤
│ 1   │ 1     │ missing │ missing │
│ 2   │ 2     │ 4       │ missing │
│ 3   │ 3     │ missing │ c       │
│ 4   │ 4     │ 2       │ d       │
│ 5   │ 5     │ 1       │ e       │

julia> dropmissing!(copy(df))
2×3 DataFrame
│ Row │ i     │ x     │ y      │
│     │ Int64 │ Int64 │ String │
├─────┼───────┼───────┼────────┤
│ 1   │ 4     │ 2     │ d      │
│ 2   │ 5     │ 1     │ e      │

julia> dropmissing!(copy(df), disallowmissing=false)
2×3 DataFrame
│ Row │ i     │ x      │ y       │
│     │ Int64 │ Int64? │ String? │
├─────┼───────┼────────┼─────────┤
│ 1   │ 4     │ 2      │ d       │
│ 2   │ 5     │ 1      │ e       │

julia> dropmissing!(copy(df), :x)
3×3 DataFrame
│ Row │ i     │ x     │ y       │
│     │ Int64 │ Int64 │ String? │
├─────┼───────┼───────┼─────────┤
│ 1   │ 2     │ 4     │ missing │
│ 2   │ 4     │ 2     │ d       │
│ 3   │ 5     │ 1     │ e       │

julia> dropmissing!(df3, [:x, :y])
2×3 DataFrame
│ Row │ i     │ x     │ y      │
│     │ Int64 │ Int64 │ String │
├─────┼───────┼───────┼────────┤
│ 1   │ 4     │ 2     │ d      │
│ 2   │ 5     │ 1     │ e      │
```
"""
function dropmissing!(df::AbstractDataFrame,
                      cols::Union{ColumnIndex, MultiColumnIndex}=:;
                      disallowmissing::Bool=true)
    delete!(df, (!).(completecases(df, cols)))
    disallowmissing && disallowmissing!(df, cols)
    df
end

"""
    filter(fun, df::AbstractDataFrame)
    filter(cols => fun, df::AbstractDataFrame)

Return a copy of data frame `df` containing only rows for which `fun`
returns `true`.

If `cols` is not specified then the predicate `fun` is passed `DataFrameRow`s.

If `cols` is specified then the predicate `fun` is passed elements of the
corresponding columns as separate positional arguments, unless `cols` is an
`AsTable` selector, in which case a `NamedTuple` of these arguments is passed.
`cols` can be any column selector ($COLUMNINDEX_STR; $MULTICOLUMNINDEX_STR), and
column duplicates are allowed if a vector of `Symbol`s, strings, or integers is
passed.

Passing `cols` leads to a more efficient execution of the operation for large data frames.

See also: [`filter!`](@ref)

# Examples
```
julia> df = DataFrame(x = [3, 1, 2, 1], y = ["b", "c", "a", "b"])
4×2 DataFrame
│ Row │ x     │ y      │
│     │ Int64 │ String │
├─────┼───────┼────────┤
│ 1   │ 3     │ b      │
│ 2   │ 1     │ c      │
│ 3   │ 2     │ a      │
│ 4   │ 1     │ b      │

julia> filter(row -> row.x > 1, df)
2×2 DataFrame
│ Row │ x     │ y      │
│     │ Int64 │ String │
├─────┼───────┼────────┤
│ 1   │ 3     │ b      │
│ 2   │ 2     │ a      │

julia> filter(:x => x -> x > 1, df)
2×2 DataFrame
│ Row │ x     │ y      │
│     │ Int64 │ String │
├─────┼───────┼────────┤
│ 1   │ 3     │ b      │
│ 2   │ 2     │ a      │

julia> filter([:x, :y] => (x, y) -> x == 1 || y == "b", df)
3×2 DataFrame
│ Row │ x     │ y      │
│     │ Int64 │ String │
├─────┼───────┼────────┤
│ 1   │ 3     │ b      │
│ 2   │ 1     │ c      │
│ 3   │ 1     │ b      │

julia> filter(AsTable(:) => nt -> nt.x == 1 || nt.y == "b", df)
3×2 DataFrame
│ Row │ x     │ y      │
│     │ Int64 │ String │
├─────┼───────┼────────┤
│ 1   │ 3     │ b      │
│ 2   │ 1     │ c      │
│ 3   │ 1     │ b      │
```
"""
Base.filter(f, df::AbstractDataFrame) = _filter_helper(df, f, eachrow(df))
Base.filter((col, f)::Pair{<:ColumnIndex}, df::AbstractDataFrame) =
    _filter_helper(df, f, df[!, col])
Base.filter((cols, f)::Pair{<:AbstractVector{Symbol}}, df::AbstractDataFrame) =
    filter([index(df)[col] for col in cols] => f, df)
Base.filter((cols, f)::Pair{<:AbstractVector{<:AbstractString}}, df::AbstractDataFrame) =
    filter([index(df)[col] for col in cols] => f, df)
Base.filter((cols, f)::Pair, df::AbstractDataFrame) =
    filter(index(df)[cols] => f, df)

function Base.filter((cols, f)::Pair{<:AbstractVector{Int}}, df::AbstractDataFrame)
    cdf = _columns(df)
    return _filter_helper(df, f, (cdf[i] for i in cols)...)
end

function _filter_helper(df::AbstractDataFrame, f, cols...)
    if length(cols) == 0
        throw(ArgumentError("At least one column must be passed to filter on"))
    end
    return df[((x...) -> f(x...)::Bool).(cols...), :]
end

function Base.filter((cols, f)::Pair{<:AsTable}, df::AbstractDataFrame)
    df_tmp = select(df, cols.cols, copycols=false)
    if ncol(df_tmp) == 0
        throw(ArgumentError("At least one column must be passed to filter on"))
    end
    return _filter_helper_astable(df, Tables.namedtupleiterator(df_tmp), f)
end

_filter_helper_astable(df::AbstractDataFrame, nti::Tables.NamedTupleIterator, f) =
    df[(x -> f(x)::Bool).(nti), :]

"""
    filter!(fun, df::AbstractDataFrame)
    filter!(cols => fun, df::AbstractDataFrame)

Remove rows from data frame `df` for which `fun` returns `false`.

If `cols` is not specified then the predicate `fun` is passed `DataFrameRow`s.

If `cols` is specified then the predicate `fun` is passed elements of the
corresponding columns as separate positional arguments, unless `cols` is an
`AsTable` selector, in which case a `NamedTuple` of these arguments is passed.
`cols` can be any column selector ($COLUMNINDEX_STR; $MULTICOLUMNINDEX_STR), and
column duplicates are allowed if a vector of `Symbol`s, strings, or integers is
passed.

Passing `cols` leads to a more efficient execution of the operation for large data frames.

See also: [`filter`](@ref)

# Examples
```
julia> df = DataFrame(x = [3, 1, 2, 1], y = ["b", "c", "a", "b"])
4×2 DataFrame
│ Row │ x     │ y      │
│     │ Int64 │ String │
├─────┼───────┼────────┤
│ 1   │ 3     │ b      │
│ 2   │ 1     │ c      │
│ 3   │ 2     │ a      │
│ 4   │ 1     │ b      │

julia> filter!(row -> row.x > 1, df);

julia> df
2×2 DataFrame
│ Row │ x     │ y      │
│     │ Int64 │ String │
├─────┼───────┼────────┤
│ 1   │ 3     │ b      │
│ 2   │ 2     │ a      │

julia> filter!(:x => x -> x == 3, df);

julia> df
1×2 DataFrame
│ Row │ x     │ y      │
│     │ Int64 │ String │
├─────┼───────┼────────┤
│ 1   │ 3     │ b      │

julia> df = DataFrame(x = [3, 1, 2, 1], y = ["b", "c", "a", "b"]);

julia> filter!([:x, :y] => (x, y) -> x == 1 || y == "b", df);

julia> df
3×2 DataFrame
│ Row │ x     │ y      │
│     │ Int64 │ String │
├─────┼───────┼────────┤
│ 1   │ 3     │ b      │
│ 2   │ 1     │ c      │
│ 3   │ 1     │ b      │

julia> df = DataFrame(x = [3, 1, 2, 1], y = ["b", "c", "a", "b"]);

julia> filter!(AsTable(:) => nt -> nt.x == 1 || nt.y == "b", df)
3×2 DataFrame
│ Row │ x     │ y      │
│     │ Int64 │ String │
├─────┼───────┼────────┤
│ 1   │ 3     │ b      │
│ 2   │ 1     │ c      │
│ 3   │ 1     │ b      │
```
"""
Base.filter!(f, df::AbstractDataFrame) = _filter!_helper(df, f, eachrow(df))
Base.filter!((col, f)::Pair{<:ColumnIndex}, df::AbstractDataFrame) =
    _filter!_helper(df, f, df[!, col])
Base.filter!((cols, f)::Pair{<:AbstractVector{Symbol}}, df::AbstractDataFrame) =
    filter!([index(df)[col] for col in cols] => f, df)
Base.filter!((cols, f)::Pair{<:AbstractVector{<:AbstractString}}, df::AbstractDataFrame) =
    filter!([index(df)[col] for col in cols] => f, df)
Base.filter!((cols, f)::Pair, df::AbstractDataFrame) =
    filter!(index(df)[cols] => f, df)

function Base.filter!((cols, f)::Pair{<:AbstractVector{Int}}, df::AbstractDataFrame)
    cdf = _columns(df)
    return _filter!_helper(df, f, (cdf[i] for i in cols)...)
end

function _filter!_helper(df::AbstractDataFrame, f, cols...)
    if length(cols) == 0
        throw(ArgumentError("At least one column must be passed to filter on"))
    end
    return delete!(df, findall(((x...) -> !(f(x...)::Bool)).(cols...)))
end

function Base.filter!((cols, f)::Pair{<:AsTable}, df::AbstractDataFrame)
    dff = select(df, cols.cols, copycols=false)
    if ncol(dff) == 0
        throw(ArgumentError("At least one column must be passed to filter on"))
    end
    return _filter!_helper_astable(df, Tables.namedtupleiterator(dff), f)
end

_filter!_helper_astable(df::AbstractDataFrame, nti::Tables.NamedTupleIterator, f) =
    delete!(df, findall((x -> !(f(x)::Bool)).(nti)))

function Base.convert(::Type{Matrix}, df::AbstractDataFrame)
    T = reduce(promote_type, (eltype(v) for v in eachcol(df)))
    return convert(Matrix{T}, df)
end

function Base.convert(::Type{Matrix{T}}, df::AbstractDataFrame) where T
    n, p = size(df)
    res = Matrix{T}(undef, n, p)
    idx = 1
    for (name, col) in pairs(eachcol(df))
        try
            copyto!(res, idx, col)
        catch err
            if err isa MethodError && err.f == convert &&
               !(T >: Missing) && any(ismissing, col)
                throw(ArgumentError("cannot convert a DataFrame containing missing " *
                                    "values to Matrix{$T} (found for column $name)"))
            else
                rethrow(err)
            end
        end
        idx += n
    end
    return res
end

Base.Matrix(df::AbstractDataFrame) = Base.convert(Matrix, df)
Base.Matrix{T}(df::AbstractDataFrame) where {T} = Base.convert(Matrix{T}, df)

Base.convert(::Type{Array}, df::AbstractDataFrame) = convert(Matrix, df)
Base.convert(::Type{Array{T}}, df::AbstractDataFrame) where {T}  = Matrix{T}(df)
Base.Array(df::AbstractDataFrame) = Matrix(df)
Base.Array{T}(df::AbstractDataFrame) where {T} = Matrix{T}(df)

"""
    nonunique(df::AbstractDataFrame)
    nonunique(df::AbstractDataFrame, cols)

Return a `Vector{Bool}` in which `true` entries indicate duplicate rows.
A row is a duplicate if there exists a prior row with all columns containing
equal values (according to `isequal`).

See also [`unique`](@ref) and [`unique!`](@ref).

# Arguments
- `df` : `AbstractDataFrame`
- `cols` : a selector specifying the column(s) to compare. Can be any column
  selector ($COLUMNINDEX_STR; $MULTICOLUMNINDEX_STR).

# Examples
```julia
df = DataFrame(i = 1:10, x = rand(10), y = rand(["a", "b", "c"], 10))
df = vcat(df, df)
nonunique(df)
nonunique(df, 1)
```
"""
function nonunique(df::AbstractDataFrame)
    if ncol(df) == 0
        throw(ArgumentError("finding duplicate rows in data frame with no" *
                            " columns is not allowed"))
    end
    gslots = row_group_slots(ntuple(i -> df[!, i], ncol(df)), Val(true))[3]
    # unique rows are the first encountered group representatives,
    # nonunique are everything else
    res = fill(true, nrow(df))
    @inbounds for g_row in gslots
        (g_row > 0) && (res[g_row] = false)
    end
    return res
end

nonunique(df::AbstractDataFrame, cols) = nonunique(select(df, cols, copycols=false))

Base.unique!(df::AbstractDataFrame) = delete!(df, findall(nonunique(df)))
Base.unique!(df::AbstractDataFrame, cols::AbstractVector) =
    delete!(df, findall(nonunique(df, cols)))
Base.unique!(df::AbstractDataFrame, cols) =
    delete!(df, findall(nonunique(df, cols)))

# Unique rows of an AbstractDataFrame.
Base.unique(df::AbstractDataFrame) = df[(!).(nonunique(df)), :]
Base.unique(df::AbstractDataFrame, cols) =
    df[(!).(nonunique(df, cols)), :]

"""
    unique(df::AbstractDataFrame)
    unique(df::AbstractDataFrame, cols)
    unique!(df::AbstractDataFrame)
    unique!(df::AbstractDataFrame, cols)

Delete duplicate rows of data frame `df`, keeping only the first occurrence of unique rows.
When `cols` is specified, the returned `DataFrame` contains complete rows,
retaining in each case the first instance for which `df[cols]` is unique.
`cols` can be any column selector ($COLUMNINDEX_STR; $MULTICOLUMNINDEX_STR).

When `unique` is called a new data frame is returned; `unique!` updates `df` in-place.

See also [`nonunique`](@ref).

# Arguments
- `df` : the AbstractDataFrame
- `cols` :  column indicator (Symbol, Int, Vector{Symbol}, Regex, etc.)
specifying the column(s) to compare.

# Examples
```julia
df = DataFrame(i = 1:10, x = rand(10), y = rand(["a", "b", "c"], 10))
df = vcat(df, df)
unique(df)   # doesn't modify df
unique(df, 1)
unique!(df)  # modifies df
```
"""
(unique, unique!)

"""
    hcat(df::AbstractDataFrame...;
         makeunique::Bool=false, copycols::Bool=true)
    hcat(df::AbstractDataFrame..., vs::AbstractVector;
         makeunique::Bool=false, copycols::Bool=true)
    hcat(vs::AbstractVector, df::AbstractDataFrame;
         makeunique::Bool=false, copycols::Bool=true)

Horizontally concatenate `AbstractDataFrames` and optionally `AbstractVector`s.

If `AbstractVector` is passed then a column name for it is automatically generated
as `:x1` by default.

If `makeunique=false` (the default) column names of passed objects must be unique.
If `makeunique=true` then duplicate column names will be suffixed
with `_i` (`i` starting at 1 for the first duplicate).

If `copycols=true` (the default) then the `DataFrame` returned by `hcat` will
contain copied columns from the source data frames.
If `copycols=false` then it will contain columns as they are stored in the
source (without copying). This option should be used with caution as mutating
either the columns in sources or in the returned `DataFrame` might lead to
the corruption of the other object.

# Example
```jldoctest
julia [DataFrame(A=1:3) DataFrame(B=1:3)]
3×2 DataFrame
│ Row │ A     │ B     │
│     │ Int64 │ Int64 │
├─────┼───────┼───────┤
│ 1   │ 1     │ 1     │
│ 2   │ 2     │ 2     │
│ 3   │ 3     │ 3     │

julia> df1 = DataFrame(A=1:3, B=1:3);

julia> df2 = DataFrame(A=4:6, B=4:6);

julia> df3 = hcat(df1, df2, makeunique=true)
3×4 DataFrame
│ Row │ A     │ B     │ A_1   │ B_1   │
│     │ Int64 │ Int64 │ Int64 │ Int64 │
├─────┼───────┼───────┼───────┼───────┤
│ 1   │ 1     │ 1     │ 4     │ 4     │
│ 2   │ 2     │ 2     │ 5     │ 5     │
│ 3   │ 3     │ 3     │ 6     │ 6     │

julia> df3.A === df1.A
false

julia> df3 = hcat(df1, df2, makeunique=true, copycols=false);

julia> df3.A === df1.A
true
```
"""
Base.hcat(df::AbstractDataFrame; makeunique::Bool=false, copycols::Bool=true) =
    DataFrame(df, copycols=copycols)
Base.hcat(df::AbstractDataFrame, x; makeunique::Bool=false, copycols::Bool=true) =
    hcat!(DataFrame(df, copycols=copycols), x,
          makeunique=makeunique, copycols=copycols)
Base.hcat(x, df::AbstractDataFrame; makeunique::Bool=false, copycols::Bool=true) =
    hcat!(x, df, makeunique=makeunique, copycols=copycols)
Base.hcat(df1::AbstractDataFrame, df2::AbstractDataFrame;
          makeunique::Bool=false, copycols::Bool=true) =
    hcat!(DataFrame(df1, copycols=copycols), df2,
          makeunique=makeunique, copycols=copycols)
Base.hcat(df::AbstractDataFrame, x, y...;
          makeunique::Bool=false, copycols::Bool=true) =
    hcat!(hcat(df, x, makeunique=makeunique, copycols=copycols), y...,
          makeunique=makeunique, copycols=copycols)
Base.hcat(df1::AbstractDataFrame, df2::AbstractDataFrame, dfn::AbstractDataFrame...;
          makeunique::Bool=false, copycols::Bool=true) =
    hcat!(hcat(df1, df2, makeunique=makeunique, copycols=copycols), dfn...,
          makeunique=makeunique, copycols=copycols)

"""
    vcat(dfs::AbstractDataFrame...;
         cols::Union{Symbol, AbstractVector{Symbol},
                     AbstractVector{<:AbstractString}}=:setequal)

Vertically concatenate `AbstractDataFrame`s.

The `cols` keyword argument determines the columns of the returned data frame:

* `:setequal`: require all data frames to have the same column names disregarding
  order. If they appear in different orders, the order of the first provided data
  frame is used.
* `:orderequal`: require all data frames to have the same column names and in the
  same order.
* `:intersect`: only the columns present in *all* provided data frames are kept.
  If the intersection is empty, an empty data frame is returned.
* `:union`: columns present in *at least one* of the provided data frames are kept.
  Columns not present in some data frames are filled with `missing` where necessary.
* A vector of `Symbol`s or strings: only listed columns are kept.
  Columns not present in some data frames are filled with `missing` where necessary.

The order of columns is determined by the order they appear in the included data
frames, searching through the header of the first data frame, then the second,
etc.

The element types of columns are determined using `promote_type`,
as with `vcat` for `AbstractVector`s.

`vcat` ignores empty data frames, making it possible to initialize an empty
data frame at the beginning of a loop and `vcat` onto it.

# Example
```jldoctest
julia> df1 = DataFrame(A=1:3, B=1:3);

julia> df2 = DataFrame(A=4:6, B=4:6);

julia> df3 = DataFrame(A=7:9, C=7:9);

julia> d4 = DataFrame();

julia> vcat(df1, df2)
6×2 DataFrame
│ Row │ A     │ B     │
│     │ Int64 │ Int64 │
├─────┼───────┼───────┤
│ 1   │ 1     │ 1     │
│ 2   │ 2     │ 2     │
│ 3   │ 3     │ 3     │
│ 4   │ 4     │ 4     │
│ 5   │ 5     │ 5     │
│ 6   │ 6     │ 6     │

julia> vcat(df1, df3, cols=:union)
6×3 DataFrame
│ Row │ A     │ B       │ C       │
│     │ Int64 │ Int64?  │ Int64?  │
├─────┼───────┼─────────┼─────────┤
│ 1   │ 1     │ 1       │ missing │
│ 2   │ 2     │ 2       │ missing │
│ 3   │ 3     │ 3       │ missing │
│ 4   │ 7     │ missing │ 7       │
│ 5   │ 8     │ missing │ 8       │
│ 6   │ 9     │ missing │ 9       │

julia> vcat(df1, df3, cols=:intersect)
6×1 DataFrame
│ Row │ A     │
│     │ Int64 │
├─────┼───────┤
│ 1   │ 1     │
│ 2   │ 2     │
│ 3   │ 3     │
│ 4   │ 7     │
│ 5   │ 8     │
│ 6   │ 9     │

julia> vcat(d4, df1)
3×2 DataFrame
│ Row │ A     │ B     │
│     │ Int64 │ Int64 │
├─────┼───────┼───────┤
│ 1   │ 1     │ 1     │
│ 2   │ 2     │ 2     │
│ 3   │ 3     │ 3     │
```

"""
Base.vcat(dfs::AbstractDataFrame...;
          cols::Union{Symbol, AbstractVector{Symbol},
                      AbstractVector{<:AbstractString}}=:setequal) =
    reduce(vcat, dfs; cols=cols)

Base.reduce(::typeof(vcat),
            dfs::Union{AbstractVector{<:AbstractDataFrame},
                       Tuple{Vararg{AbstractDataFrame}}};
            cols::Union{Symbol, AbstractVector{Symbol},
                        AbstractVector{<:AbstractString}}=:setequal) =
    _vcat([df for df in dfs if ncol(df) != 0]; cols=cols)

function _vcat(dfs::AbstractVector{<:AbstractDataFrame};
               cols::Union{Symbol, AbstractVector{Symbol},
                           AbstractVector{<:AbstractString}}=:setequal)

    isempty(dfs) && return DataFrame()
    # Array of all headers
    allheaders = map(names, dfs)
    # Array of unique headers across all data frames
    uniqueheaders = unique(allheaders)
    # All symbols present across all headers
    unionunique = union(uniqueheaders...)
    # List of symbols present in all dataframes
    intersectunique = intersect(uniqueheaders...)

    if cols === :orderequal
        header = unionunique
        if length(uniqueheaders) > 1
            throw(ArgumentError("when `cols=:orderequal` all data frames need to " *
                                "have the same column names and be in the same order"))
        end
    elseif cols === :setequal || cols === :equal
        # an explicit error is thrown as :equal was supported in the past
        if cols === :equal
            throw(ArgumentError("`cols=:equal` is not supported. " *
                                "Use `:setequal` instead."))
        end

        header = unionunique
        coldiff = setdiff(unionunique, intersectunique)

        if !isempty(coldiff)
            # if any DataFrames are a full superset of names, skip them
            filter!(u -> !issetequal(u, header), uniqueheaders)
            estrings = map(enumerate(uniqueheaders)) do (i, head)
                matching = findall(h -> head == h, allheaders)
                headerdiff = setdiff(coldiff, head)
                cols = join(headerdiff, ", ", " and ")
                args = join(matching, ", ", " and ")
                return "column(s) $cols are missing from argument(s) $args"
            end
            throw(ArgumentError(join(estrings, ", ", ", and ")))
        end
    elseif cols === :intersect
        header = intersectunique
    elseif cols === :union
        header = unionunique
    elseif cols isa Symbol
        throw(ArgumentError("Invalid `cols` value :$cols. " *
                            "Only `:orderequal`, `:setequal`, `:intersect`, " *
                            "`:union`, or a vector of column names is allowed."))
    elseif cols isa AbstractVector{Symbol}
        header = cols
    else
        @assert cols isa AbstractVector{<:AbstractString}
        header = Symbol.(cols)
    end

    length(header) == 0 && return DataFrame()
    all_cols = Vector{AbstractVector}(undef, length(header))
    for (i, name) in enumerate(header)
        newcols = map(dfs) do df
            if hasproperty(df, name)
                return df[!, name]
            else
                Iterators.repeated(missing, nrow(df))
            end
        end

        lens = map(length, newcols)
        T = mapreduce(eltype, promote_type, newcols)
        all_cols[i] = Tables.allocatecolumn(T, sum(lens))
        offset = 1
        for j in 1:length(newcols)
            copyto!(all_cols[i], offset, newcols[j])
            offset += lens[j]
        end
    end
    return DataFrame(all_cols, header, copycols=false)
end

"""
    repeat(df::AbstractDataFrame; inner::Integer = 1, outer::Integer = 1)

Construct a data frame by repeating rows in `df`. `inner` specifies how many
times each row is repeated, and `outer` specifies how many times the full set
of rows is repeated.

# Example
```jldoctest
julia> df = DataFrame(a = 1:2, b = 3:4)
2×2 DataFrame
│ Row │ a     │ b     │
│     │ Int64 │ Int64 │
├─────┼───────┼───────┤
│ 1   │ 1     │ 3     │
│ 2   │ 2     │ 4     │

julia> repeat(df, inner = 2, outer = 3)
12×2 DataFrame
│ Row │ a     │ b     │
│     │ Int64 │ Int64 │
├─────┼───────┼───────┤
│ 1   │ 1     │ 3     │
│ 2   │ 1     │ 3     │
│ 3   │ 2     │ 4     │
│ 4   │ 2     │ 4     │
│ 5   │ 1     │ 3     │
│ 6   │ 1     │ 3     │
│ 7   │ 2     │ 4     │
│ 8   │ 2     │ 4     │
│ 9   │ 1     │ 3     │
│ 10  │ 1     │ 3     │
│ 11  │ 2     │ 4     │
│ 12  │ 2     │ 4     │
```
"""
function Base.repeat(df::AbstractDataFrame; inner::Integer = 1, outer::Integer = 1)
    inner < 0 && throw(ArgumentError("inner keyword argument must be non-negative"))
    outer < 0 && throw(ArgumentError("outer keyword argument must be non-negative"))
    return mapcols(x -> repeat(x, inner = inner, outer = outer), df)
end

"""
    repeat(df::AbstractDataFrame, count::Integer)

Construct a data frame by repeating each row in `df` the number of times
specified by `count`.

# Example
```jldoctest
julia> df = DataFrame(a = 1:2, b = 3:4)
2×2 DataFrame
│ Row │ a     │ b     │
│     │ Int64 │ Int64 │
├─────┼───────┼───────┤
│ 1   │ 1     │ 3     │
│ 2   │ 2     │ 4     │

julia> repeat(df, 2)
4×2 DataFrame
│ Row │ a     │ b     │
│     │ Int64 │ Int64 │
├─────┼───────┼───────┤
│ 1   │ 1     │ 3     │
│ 2   │ 2     │ 4     │
│ 3   │ 1     │ 3     │
│ 4   │ 2     │ 4     │
```
"""
function Base.repeat(df::AbstractDataFrame, count::Integer)
    count < 0 && throw(ArgumentError("count must be non-negative"))
    return mapcols(x -> repeat(x, count), df)
end

##############################################################################
##
## Hashing
##
##############################################################################

const hashdf_seed = UInt == UInt32 ? 0xfd8bb02e : 0x6215bada8c8c46de

function Base.hash(df::AbstractDataFrame, h::UInt)
    h += hashdf_seed
    h += hash(size(df))
    for i in 1:size(df, 2)
        h = hash(df[!, i], h)
    end
    return h
end

Base.parent(adf::AbstractDataFrame) = adf
Base.parentindices(adf::AbstractDataFrame) = axes(adf)

## Documentation for methods defined elsewhere

function nrow end
function ncol end

"""
    nrow(df::AbstractDataFrame)
    ncol(df::AbstractDataFrame)

Return the number of rows or columns in an `AbstractDataFrame` `df`.

See also [`size`](@ref).

**Examples**

```jldoctest
julia> df = DataFrame(i = 1:10, x = rand(10), y = rand(["a", "b", "c"], 10));

julia> size(df)
(10, 3)

julia> nrow(df)
10

julia> ncol(df)
3
```

"""
(nrow, ncol)

"""
    disallowmissing(df::AbstractDataFrame, cols=:; error::Bool=true)

Return a copy of data frame `df` with columns `cols` converted
from element type `Union{T, Missing}` to `T` to drop support for missing values.

`cols` can be any column selector ($COLUMNINDEX_STR; $MULTICOLUMNINDEX_STR).

If `cols` is omitted all columns in the data frame are converted.

If `error=false` then columns containing a `missing` value will be skipped instead
of throwing an error.

**Examples**

```jldoctest
julia> df = DataFrame(a=Union{Int,Missing}[1,2])
2×1 DataFrame
│ Row │ a      │
│     │ Int64? │
├─────┼────────┤
│ 1   │ 1      │
│ 2   │ 2      │

julia> disallowmissing(df)
2×1 DataFrame
│ Row │ a     │
│     │ Int64 │
├─────┼───────┤
│ 1   │ 1     │
│ 2   │ 2     │

julia> df = DataFrame(a=[1,missing])
2×2 DataFrame
│ Row │ a       │ b      │
│     │ Int64?  │ Int64? │
├─────┼─────────┼────────┤
│ 1   │ 1       │ 1      │
│ 2   │ missing │ 2      │

julia> disallowmissing(df, error=false)
2×2 DataFrame
│ Row │ a       │ b     │
│     │ Int64?  │ Int64 │
├─────┼─────────┼───────┤
│ 1   │ 1       │ 1     │
│ 2   │ missing │ 2     │
```
"""
function Missings.disallowmissing(df::AbstractDataFrame,
                                  cols::Union{ColumnIndex, MultiColumnIndex}=:;
                                  error::Bool=true)
    idxcols = Set(index(df)[cols])
    newcols = AbstractVector[]
    for i in axes(df, 2)
        x = df[!, i]
        if i in idxcols
            if !error && Missing <: eltype(x) && any(ismissing, x)
                y = x
            else
                y = disallowmissing(x)
            end
            push!(newcols, y === x ? copy(y) : y)
        else
            push!(newcols, copy(x))
        end
    end
    return DataFrame(newcols, _names(df), copycols=false)
end

"""
    allowmissing(df::AbstractDataFrame, cols=:)

Return a copy of data frame `df` with columns `cols` converted
to element type `Union{T, Missing}` from `T` to allow support for missing values.

`cols` can be any column selector ($COLUMNINDEX_STR; $MULTICOLUMNINDEX_STR).

If `cols` is omitted all columns in the data frame are converted.

**Examples**

```jldoctest
julia> df = DataFrame(a=[1,2])
2×1 DataFrame
│ Row │ a     │
│     │ Int64 │
├─────┼───────┤
│ 1   │ 1     │
│ 2   │ 2     │

julia> allowmissing(df)
2×1 DataFrame
│ Row │ a      │
│     │ Int64? │
├─────┼────────┤
│ 1   │ 1      │
│ 2   │ 2      │
```
"""
function Missings.allowmissing(df::AbstractDataFrame,
                               cols::Union{ColumnIndex, MultiColumnIndex}=:)
    idxcols = Set(index(df)[cols])
    newcols = AbstractVector[]
    for i in axes(df, 2)
        x = df[!, i]
        if i in idxcols
            y = allowmissing(x)
            push!(newcols, y === x ? copy(y) : y)
        else
            push!(newcols, copy(x))
        end
    end
    return DataFrame(newcols, _names(df), copycols=false)
end

"""
    categorical(df::AbstractDataFrame, cols=Union{AbstractString, Missing};
                compress::Bool=false)

Return a copy of data frame `df` with columns `cols` converted to `CategoricalVector`.

`cols` can be any column selector ($COLUMNINDEX_STR; $MULTICOLUMNINDEX_STR)
or a `Type`.

If `categorical` is called with the `cols` argument being a `Type`, then
all columns whose element type is a subtype of this type
(by default `Union{AbstractString, Missing}`) will be converted to categorical.

If the `compress` keyword argument is set to `true` then the created
`CategoricalVector`s will be compressed.

All created `CategoricalVector`s are unordered.

**Examples**

```jldoctest
julia> df = DataFrame(a=[1,2], b=["a","b"])
2×2 DataFrame
│ Row │ a     │ b      │
│     │ Int64 │ String │
├─────┼───────┼────────┤
│ 1   │ 1     │ a      │
│ 2   │ 2     │ b      │

julia> categorical(df)
2×2 DataFrame
│ Row │ a     │ b    │
│     │ Int64 │ Cat… │
├─────┼───────┼──────┤
│ 1   │ 1     │ a    │
│ 2   │ 2     │ b    │

julia> categorical(df, :)
2×2 DataFrame
│ Row │ a    │ b    │
│     │ Cat… │ Cat… │
├─────┼──────┼──────┤
│ 1   │ 1    │ a    │
│ 2   │ 2    │ b    │
```
"""
function CategoricalArrays.categorical(df::AbstractDataFrame,
                                       cols::Union{ColumnIndex, MultiColumnIndex};
                                       compress::Bool=false)
    idxcols = Set(index(df)[cols])
    newcols = AbstractVector[]
    for i in axes(df, 2)
        x = df[!, i]
        if i in idxcols
            # categorical always copies
            push!(newcols, categorical(x, compress=compress))
        else
            push!(newcols, copy(x))
        end
    end
    DataFrame(newcols, _names(df), copycols=false)
end

function CategoricalArrays.categorical(df::AbstractDataFrame,
                                       cols::Type=Union{AbstractString, Missing};
                                       compress::Bool=false)
    newcols = AbstractVector[]
    for i in axes(df, 2)
        x = df[!, i]
        if eltype(x) <: cols
            # categorical always copies
            push!(newcols, categorical(x, compress=compress))
        else
            push!(newcols, copy(x))
        end
    end
    DataFrame(newcols, _names(df), copycols=false)
end

"""
    flatten(df::AbstractDataFrame, cols)

When columns `cols` of data frame `df` have iterable elements that define
`length` (for example a `Vector` of `Vector`s), return a `DataFrame` where each
element of each `col` in `cols` is flattened, meaning the column corresponding
to `col` becomes a longer vector where the original entries are concatenated.
Elements of row `i` of `df` in columns other than `cols` will be repeated
according to the length of `df[i, col]`. These lengths must therefore be the
same for each `col` in `cols`, or else an error is raised. Note that these
elements are not copied, and thus if they are mutable changing them in the
returned `DataFrame` will affect `df`.

`cols` can be any column selector ($COLUMNINDEX_STR; $MULTICOLUMNINDEX_STR).

# Examples

```
julia> df1 = DataFrame(a = [1, 2], b = [[1, 2], [3, 4]], c = [[5, 6], [7, 8]])
2×3 DataFrame
│ Row │ a     │ b      │ c      │
│     │ Int64 │ Array… │ Array… │
├─────┼───────┼────────┼────────┤
│ 1   │ 1     │ [1, 2] │ [5, 6] │
│ 2   │ 2     │ [3, 4] │ [7, 8] │

julia> flatten(df1, :b)
4×3 DataFrame
│ Row │ a     │ b     │ c      │
│     │ Int64 │ Int64 │ Array… │
├─────┼───────┼───────┼────────┤
│ 1   │ 1     │ 1     │ [5, 6] │
│ 2   │ 1     │ 2     │ [5, 6] │
│ 3   │ 2     │ 3     │ [7, 8] │
│ 4   │ 2     │ 4     │ [7, 8] │

julia> flatten(df1, [:b, :c])
4×3 DataFrame
│ Row │ a     │ b     │ c     │
│     │ Int64 │ Int64 │ Int64 │
├─────┼───────┼───────┼───────┤
│ 1   │ 1     │ 1     │ 5     │
│ 2   │ 1     │ 2     │ 6     │
│ 3   │ 2     │ 3     │ 7     │
│ 4   │ 2     │ 4     │ 8     │

julia> df2 = DataFrame(a = [1, 2], b = [("p", "q"), ("r", "s")])
2×2 DataFrame
│ Row │ a     │ b          │
│     │ Int64 │ Tuple…     │
├─────┼───────┼────────────┤
│ 1   │ 1     │ ("p", "q") │
│ 2   │ 2     │ ("r", "s") │

julia> flatten(df2, :b)
4×2 DataFrame
│ Row │ a     │ b      │
│     │ Int64 │ String │
├─────┼───────┼────────┤
│ 1   │ 1     │ p      │
│ 2   │ 1     │ q      │
│ 3   │ 2     │ r      │
│ 4   │ 2     │ s      │

julia> df3 = DataFrame(a = [1, 2], b = [[1, 2], [3, 4]], c = [[5, 6], [7]])
2×3 DataFrame
│ Row │ a     │ b      │ c      │
│     │ Int64 │ Array… │ Array… │
├─────┼───────┼────────┼────────┤
│ 1   │ 1     │ [1, 2] │ [5, 6] │
│ 2   │ 2     │ [3, 4] │ [7]    │

julia> flatten(df3, [:b, :c])
ERROR: ArgumentError: Lengths of iterables stored in columns :b and :c
are not the same in row 2
```
"""
function flatten(df::AbstractDataFrame,
                 cols::Union{ColumnIndex, MultiColumnIndex})
    _check_consistency(df)

    idxcols = index(df)[cols]
    isempty(idxcols) && return copy(df)
    col1 = first(idxcols)
    lengths = length.(df[!, col1])
    for col in idxcols
        v = df[!, col]
        if any(x -> length(x[1]) != x[2], zip(v, lengths))
            r = findfirst(x -> x != 0, length.(v) .- lengths)
            colnames = _names(df)
            throw(ArgumentError("Lengths of iterables stored in columns :$(colnames[col1])" *
                                " and :$(colnames[col]) are not the same in row $r"))
        end
    end

    new_df = similar(df[!, Not(cols)], sum(lengths))
    for name in _names(new_df)
        repeat_lengths!(new_df[!, name], df[!, name], lengths)
    end
    length(idxcols) > 1 && sort!(idxcols)
    for col in idxcols
        col_to_flatten = df[!, col]
        flattened_col = col_to_flatten isa AbstractVector{<:AbstractVector} ?
            reduce(vcat, col_to_flatten) :
            collect(Iterators.flatten(col_to_flatten))

        insertcols!(new_df, col, _names(df)[col] => flattened_col)
    end

    return new_df
end

function repeat_lengths!(longnew::AbstractVector, shortold::AbstractVector,
                         lengths::AbstractVector{Int})
    counter = 1
    @inbounds for i in eachindex(shortold)
        l = lengths[i]
        longnew[counter:(counter + l - 1)] .= Ref(shortold[i])
        counter += l
    end
end


text:
module BSplines

using IterTools
# using OffsetArrays

export BSpline, calc, gen_matrix, predict

struct BSpline
    p
    n
    t
end

function BSpline(p::Int, t::AbstractArray{T}) where T <: Real
    # number of knots t[i]
    n = size(t, 1)

    # extend knots by p on both sides of t
    tt = [repeat(t[1:1], p)..., t..., repeat(t[end:end], p)...]

    BSpline(p, n, tt)
end

function calc(b::BSpline, i::Int, p::Int, x::T) where T <: Real
    if (i + p + 1 > b.n + 2 * b.p) || (i < 1) || !(b.t[i] <= x <= b.t[i+p+1])
        return zero(T)
    elseif p === b.p && i === b.n + p - 1 && x == b.t[end]
        return one(T)
    end

    if p === 0
        if (i + 1 <= b.n + b.p) && (b.t[i] <= x < b.t[i+1])
            one(T)
        else
            zero(T)
        end
    else
        b1 = calc(b, i, p - 1, x)
        a1 = b1 === zero(T) ? zero(T) : (x - b.t[i]) / (b.t[i + p] - b.t[i])

        b2 = calc(b, i + 1, p - 1, x)
        a2 = b2 === zero(T) ? zero(T) : (b.t[i + p + 1] - x) / (b.t[i + p + 1] - b.t[i + 1])

        a1 * b1 + a2 * b2
    end
end

calc(b::BSpline, i::Int, x::T) where T <: Real = calc(b, i, b.p, x)

function gen_matrix(b::BSpline, xs::AbstractArray{T}) where T <: Real
    [
        calc(b, i, x) for x in xs, i in 1:(b.n + b.p - 1)
    ]
end

function predict(b::BSpline, w::AbstractArray{T}, x::T) where T <: Real
    parts = partition(b.t, 2, 1) |> collect
    k = findfirst(i -> first(i) <= x < last(i), parts)
    if isnothing(k)
        return zero(T)
    end
    sum(w[i] * calc(b, i, b.p, x) for i in (k - b.p):k)
end

predict(b::BSpline, x::T) where T <: Real = predict(b, ones(T, b.n + b.p - 1), x)

end  # module

ext: jl


text:
immutable ArgsIndex
    index::Dict{Symbol, Int}
    args::Vector{Set{Symbol}}
end
ArgsIndex() = ArgsIndex(Dict{Symbol,Int}(), Vector{Set{Symbol}}())

Base.getindex(ai::ArgsIndex, t::Symbol) = ai.args[ai.index[t]]
function Base.setindex!(ai::ArgsIndex, s, t::Symbol)
    i = get!(ai.index, t, length(ai.index)+1)
    i == length(ai.index) ? push!(ai.args, s) : (ai.args[i] = s)
    return s
end
Base.get!(ai::ArgsIndex, t::Symbol) = haskey(ai.index, t) ? (ai[t]) : (ai[t] = Set{Symbol}())

"""
    abstract QueryHelper

Leaf subtypes `T <: QueryHelper` contain data extracted from query arguments
and stored in fields of `T` as resources for collection machineries.
"""
abstract Verb

immutable Select{F} <: Verb
    res_field::Symbol
    f::F
    ai::ArgsIndex
end
immutable Filter{F} <: Verb
    f::F
    ai::ArgsIndex
end
immutable OrderBy{F} <: Verb
    f::F
    ai::ArgsIndex
end
immutable GroupBy{F} <: Verb
    is_predicate::Bool
    f::F
    ai::ArgsIndex
end
immutable Summarize{F,G} <: Verb
    res_field::Symbol
    f::F
    g::G
    ai::ArgsIndex
end
immutable LeftJoin{F,G} <: Verb
    fs::Tuple{F,G}
    ais::Tuple{ArgsIndex,ArgsIndex}
end
immutable OuterJoin{F,G} <: Verb
    fs::Tuple{F,G}
    ais::Tuple{ArgsIndex,ArgsIndex}
end
immutable InnerJoin{F,G} <: Verb
    fs::Tuple{F,G}
    ais::Tuple{ArgsIndex,ArgsIndex}
end
immutable CrossJoin{F,G} <: Verb
    fs::Tuple{F,G}
    ais::Tuple{ArgsIndex,ArgsIndex}
end


text:
struct LatinFeaturization <: SeaPearl.AbstractFeaturization end

#Proposition of featurization for the latin square problem
#the global feature completes the state of completion of the board at each step
#it also adds the value given to a variable when it changes

function SeaPearl.featurize(sr::SeaPearl.DefaultStateRepresentation{LatinFeaturization,TS}) where TS
    N = latin_generator.N
    p = latin_generator.p
    g = sr.cplayergraph
    features = zeros(Float32, 7, nv(g))
    for i in 1:nv(g)
        cp_vertex = SeaPearl.cpVertexFromIndex(g, i)
        if isa(cp_vertex, SeaPearl.VariableVertex)
            features[1,i]=1
            id = cp_vertex.variable.id
            t = split(id,['_'])[2]
            a,b = parse(Int, split(t,",")[1]), parse(Int, split(t,",")[2]) #coordinates on the board
            features[2,i],features[3,i] = a,b
        elseif isa(cp_vertex, SeaPearl.ConstraintVertex)
            if isa(cp_vertex.constraint, SeaPearl.EqualConstant)
                features[5,i] = cp_vertex.constraint.v
            else
                features[6,i] = 1
            end
        elseif isa(cp_vertex, SeaPearl.ValueVertex)
            features[7,i] = cp_vertex.value
        end
    end
    features
end

function SeaPearl.global_featurize(sr::SeaPearl.DefaultStateRepresentation{LatinFeaturization,TS}) where TS
    N = latin_generator.N
    p = latin_generator.p
    g = sr.cplayergraph
    feature = zeros(Float32, N^2)
    for i in 1:nv(g)
        cp_vertex = SeaPearl.cpVertexFromIndex(g, i)
        if isa(cp_vertex, SeaPearl.ConstraintVertex)&&isa(cp_vertex.constraint, SeaPearl.EqualConstant)
            x = cp_vertex.constraint.x
            id=x.id
            v = cp_vertex.constraint.v
            t = split(id,['_'])[2]
            a,b = parse(Int, split(t,",")[1]), parse(Int, split(t,",")[2])
            feature[(a-1)*N + b] = v
        end
    end
    feature
end

function SeaPearl.update_features!(sr::SeaPearl.DefaultStateRepresentation{LatinFeaturization,TS}, model::SeaPearl.CPModel) where TS
    N = latin_generator.N
    p = latin_generator.p
    g = sr.cplayergraph
    vars_branchable = SeaPearl.branchable_variables(model)
    globalFeature = sr.globalFeatures
    features = sr.nodeFeatures
    for i in 1:nv(g)
        cp_vertex = SeaPearl.cpVertexFromIndex(g, i)
        if isa(cp_vertex, SeaPearl.VariableVertex)
            var = cp_vertex.variable
            id = cp_vertex.variable.id
            t = split(id,['_'])[2]
            a,b = parse(Int, split(t,",")[1]), parse(Int, split(t,",")[2])
            if SeaPearl.isbound(var)
                features[4, i] = SeaPearl.assignedValue(var)
                globalFeature[(a-1)*N + b] = SeaPearl.assignedValue(var)
            else
                features[4, i] = 0
                globalFeature[(a-1)*N + b] = 0
            end
        end
    end
end

function SeaPearl.feature_length(::Type{SeaPearl.DefaultStateRepresentation{LatinFeaturization, TS}}) where TS
    return 7
end

function SeaPearl.global_feature_length(::Type{SeaPearl.DefaultStateRepresentation{LatinFeaturization, TS}}) where TS
    return latin_generator.N*latin_generator.N
end

max_stars_repo_name: corail-research/SeaPearlZoo

max_issues_repo_name: corail-research/SeaPearlZoo

max_forks_repo_name: corail-research/SeaPearlZoo


text:
type RenderText <: Drawable
    ptr::Ptr{Void}
    _font::Font

    function RenderText(ptr::Ptr{Void})
        t = new(ptr)
        finalizer(t, destroy)
        t
    end
end

const TextRegular = 0
const TextBold = 1 << 0
const TextItalic = 1 << 1
const TextUnderlined = 1 << 2
const TextStrikeThrough = 1 << 3

function RenderText()
    r = RenderText(ccall((:sfText_create, libcsfml_graphics), Ptr{Void}, ()))
    set_font(r, Font(joinpath(dirname(@__FILE__),"..","..","..","assets","arial.ttf")))
    r
end

function RenderText(value::AbstractString)
    r = RenderText()
    set_string(r, value)
    r
end

function copy(text::RenderText)
    return RenderText(ccall((:sfText_copy, libcsfml_graphics), Ptr{Void}, (Ptr{Void},), text.ptr))
end

function destroy(text::RenderText)
    ccall((:sfText_destroy, libcsfml_graphics), Void, (Ptr{Void},), text.ptr)
end

function set_position(text::RenderText, pos::Vector2f)
    ccall((:sfText_setPosition, libcsfml_graphics), Void, (Ptr{Void}, Vector2f,), text.ptr, pos)
end

function set_rotation(text::RenderText, angle::Real)
    ccall((:sfText_setRotation, libcsfml_graphics), Void, (Ptr{Void}, Cfloat,), text.ptr, angle)
end

function set_scale(text::RenderText, scale::Vector2f)
    ccall((:sfText_setScale, libcsfml_graphics), Void, (Ptr{Void}, Vector2f), text.ptr, scale)
end

function set_origin(text::RenderText, origin::Vector2f)
    ccall((:sfText_setOrigin, libcsfml_graphics), Void, (Ptr{Void}, Vector2f,), text.ptr, origin)
end

function get_position(text::RenderText)
    return ccall((:sfText_getPosition, libcsfml_graphics), Vector2f, (Ptr{Void},), text.ptr)
end

function get_rotation(text::RenderText)
    ccall((:sfText_getRotation, libcsfml_graphics), Cfloat, (Ptr{Void},), text.ptr)
end

function get_scale(text::RenderText)
    ccall((:sfText_getScale, libcsfml_graphics), Vector2f, (Ptr{Void},), text.ptr)
end

function get_origin(text::RenderText)
    ccall((:sfText_getOrigin, libcsfml_graphics), Vector2f, (Ptr{Void},), text.ptr)
end

function move(text::RenderText, offsets::Vector2f)
    ccall((:sfText_move, libcsfml_graphics), Void, (Ptr{Void}, Vector2f,), text.ptr, offsets)
end

function rotate(text::RenderText, angle::Real)
    ccall((:sfText_rotate, libcsfml_graphics), Void, (Ptr{Void}, Cfloat,), text.ptr, angle)
end

function scale(text::RenderText, factors::Vector2f)
    ccall((:sfText_scale, libcsfml_graphics), Void, (Ptr{Void}, Vector2f,), text.ptr, factors)
end

function set_string(text::RenderText, string::AbstractString)
    ccall((:sfText_setString, libcsfml_graphics), Void, (Ptr{Void}, Ptr{Cchar},), text.ptr, string)
end

function set_font(text::RenderText, font::Font)
    ccall((:sfText_setFont, libcsfml_graphics), Void, (Ptr{Void}, Ptr{Void},), text.ptr, font.ptr)
    text._font = font
end

function set_charactersize(text::RenderText, size::Integer)
    ccall((:sfText_setCharacterSize, libcsfml_graphics), Void, (Ptr{Void}, UInt32,), text.ptr, size)
end

function set_style(text::RenderText, style::Integer)
    ccall((:sfText_setStyle, libcsfml_graphics), Void, (Ptr{Void}, UInt32,), text.ptr, style)
end

function set_color(text::RenderText, color::Color)
    ccall((:sfText_setColor, libcsfml_graphics), Void, (Ptr{Void}, Color,), text.ptr, color)
end

function set_style(text::RenderText, style::UInt32)
    ccall((:sfText_setStyle, libcsfml_graphics), Void, (Ptr{Void}, UInt32,), text.ptr, style)
end

function get_string(text::RenderText)
    unsafe_string(ccall((:sfText_getString, libcsfml_graphics), Ptr{Cchar}, (Ptr{Void},), text.ptr))
end

function get_font(text::RenderText)
    Font(ccall((:sfText_getFont, libcsfml_graphics), Ptr{Void}, (Ptr{Void},), text.ptr))
end

function get_charactersize(text::RenderText)
    Int(ccall((:sfText_getCharacterSize, libcsfml_graphics), UInt32, (Ptr{Void},), text.ptr))
end

function get_style(text::RenderText)
    Int(ccall((:sfText_getStyle, libcsfml_graphics), UInt32, (Ptr{Void},), text.ptr))
end

function get_color(text::RenderText)
    ccall((:sfText_getColor, libcsfml_graphics), Color, (Ptr{Void},), text.ptr)
end

function find_character_pos(text::RenderText, index::Integer)
    ccall((:sfText_findCharacterPos, libcsfml_graphics), Vector2f, (Ptr{Void}, Csize_t,), text.ptr, index)
end

function get_localbounds(text::RenderText)
    ccall((:sfText_getLocalBounds, libcsfml_graphics), FloatRect, (Ptr{Void},), text.ptr)
end

function get_globalbounds(text::RenderText)
    ccall((:sfText_getGlobalBounds, libcsfml_graphics), FloatRect, (Ptr{Void},), text.ptr)
end


text:
@testset "condition 1" begin
    t = trace()
    n_datapoints = 2
    program!(t, n_datapoints)
    @test t[(:data, 1)].interpretation == NONSTANDARD
    cp = condition(program!, Dict((:data, 1) => 4))
    t, _ = cp(t, n_datapoints)
    @test t[(:data, 1)].interpretation == CONDITIONED
    @test t[(:data, 1)].value == 4
    @info "(:data, 1) = $(t[(:data, 1)])"
end

function program2!(t :: Trace, data)
    loc = sample(t, :loc, Normal())
    scale = sample(t, :scale, Gamma(2.0, 2.0))
    log_rate = sample(t, :log_rate, Normal(loc, scale))
    obs = Array{Int64, 1}(undef, length(data))
    for (i, d) in enumerate(data)
        obs[i] = observe(t, (:data, i), Poisson(exp(log_rate)), d)
    end
    obs
end

@testset "observing data" begin
    data = [1, 1, 2, 1, 5]
    t = trace()
    obs = program2!(t, data)
    logprob!(t)
    @test obs == data
    # log likelihood is always >= log joint density
    log_joint = logprob(t)
    log_likelihood = loglikelihood(t)
    @info "log p(x, z) = $log_joint"
    @info "log p(x | z) = $log_likelihood"
    @test log_likelihood >= log_joint

    # copy the old trace and demonstrate modification
    t_copy = deepcopy(t)
    obs = program2!(t, data)
    @info ":loc node in copied original: $(t_copy[:loc])"
    @info ":loc node in modified original: $(t[:loc])"
    @test t[:loc].value != t_copy[:loc].value
end

@testset "replay" begin
    data = [1, 1, 2, 1, 5]
    # trace the program
    t = trace()
    obs = program2!(t, data)
    logprob!(t)
    # modify a trace element
    t2 = deepcopy(t)
    t2[:scale] = node(2.0, :scale, t[:scale].dist, false, NONSTANDARD)
    @test t[:scale].value != t2[:scale].value
    @test t[:scale].dist == t2[:scale].dist
    # replay the program using the new trace
    t_new, replayed_f = replay(program2!, t2)
    r_obs = replayed_f(data)
    logprob!(t_new)
    @test r_obs == data
    @test t[:scale].dist == t_new[:scale].dist
    @test t[:log_rate].dist != t_new[:log_rate].dist
    @test t_new[:scale].value == 2.0
    @info "Old scale node: $(t[:scale])"
    @info "New scale node: $(t_new[:scale])"
    @info "Old log_rate node: $(t[:log_rate])"
    @info "New log_rate node: $(t_new[:log_rate])"
end

@testset "block" begin
    data = [1, 1, 2, 1, 5]
    t = trace()
    obs = program2!(t, data)
    logprob!(t)
    @test :loc in keys(t)

    t_new, blocked_f = block(program2!, t, (:loc,))
    r_obs = blocked_f(data)
    logprob!(t_new)
    @test !(:loc in keys(t_new))
end

function plate_program!(t :: Trace, data)
    loc = sample(t, :loc, Normal())
    scale = sample(t, :scale, Gamma(2.0, 2.0))
    log_rate = sample(t, :log_rate, Normal(loc, scale))
    plate(t, observe, :data, Poisson(exp(log_rate)), data)
end

@testset "plate observe" begin
    data = [1, 2, 3]
    t = trace()
    plate_program!(t, data)
    @info "Trace: $t"
    @test typeof(t[:data].value) == Vector{Int64}
    d = Poisson(exp(t[:log_rate].value))
    @test isapprox(t[:data].logprob_sum, sum(logpdf(d, i) for i in data))
end

@testset "plate latent" begin
    t = trace()
    plate(t, sample, "address", Normal(), 10)
    @info "Trace: $t"
    @test isapprox(t["address"].logprob_sum, sum(logpdf(Normal(), r) for r in t["address"].value))
end

@testset "plate mh inference" begin
    data = [1, 3, 1, 5, 4]
    results = mh(plate_program!; params = (data,))
    @info "Single results trace: $(results.traces[end])"
    @test typeof(results.traces[end][:data].value) == Vector{Int64}
end

@testset "plate replay" begin
    t = trace()
    f = (t,) -> plate(t, sample, "address", Normal(), 3)
    plate(t, sample, "address", Normal(), 3)
    @info "Original trace: $t"
    (new_t, g) = replay(f, t)
    @info "Replayed trace before function exec: $new_t"
    g()
    @info "Replayed trace after function exec: $new_t"
end

function expanded_plate_model(t, data)
    hl = sample(t, "hl", Normal())
    locs = plate(t, sample, "locs", Normal(hl, 1.0), length(data))
    plate(t, observe, "data", Normal, data, (locs, ones(length(data))))
end

@testset "expanded obs plate 1" begin
    t = trace()
    data = randn(3)
    expanded_plate_model(t, data)
    @info t
end

locs_proposal(t0, t1, params...) = propose(t1, "locs", MvNormal(t0["locs"].value, 1.0))
hl_proposal(t0, t1, params...) = propose(t1, "hl", Normal(t0["hl"].value, 1.0))

@testset "expanded obs plate 2" begin
    data = randn(10)
    results = mh(expanded_plate_model, [locs_proposal, hl_proposal]; params = (data,))
end


text:
"""
```
measurement(m::PoolModel{T}) where {T<:AbstractFloat}
```

Assign measurement equation for Dynamic Pools

```
p(y_t) = λ_t * p_1 + (1 - λ_t) * p_2
```

where

```
λ_t = weight assigned to model 1
p_1 = predictive density according to model 1
p_2 = predictive density according to model 2
```

Equal weights fixes λ = 1/2, and static weights computes the likelihood
as if λ is time-invariant. For Bayesian Model Averaging,
we compute the average according to weights computed according to BMA.

The outputs are a function that computes the average of
passed predictive densities and a degenerate measurement error Distribution.
"""
function measurement(m::PoolModel{T}) where {T<:AbstractFloat}
    obs = m.observables
    weight_type = get_setting(m, :weight_type)

    # Assumes λ to be weight on the first model
    F_u = DiscreteUniform(0,0)
    if weight_type == :dynamic
        Ψ_dynamic_pm(x::Vector{Float64}, data::Vector{Float64}) = dot(data, x)
        return Ψ_dynamic_pm, F_u
    elseif weight_type == :equal
        equal_wt = m[:λ].value
        Ψ_equal_pm(x::Vector{Float64}, data::Vector{Float64}) = dot(data, equal_wt .* ones(2))
        return Ψ_equal_pm, F_u
    elseif weight_type == :static
        Ψ_static_pm(x::Vector{Float64}, data::Vector{Float64}) = dot(data, x)
        return Ψ_static_pm, F_u
    elseif weight_type == :bma
        Ψ_bma_pm(x::Vector{Float64}, data::Vector{Float64}) = dot(data, x)
        return Ψ_bma_pm, F_u
    end
end


text:
# ---
# title: 81. Search in Rotated Sorted Array II
# id: problem81
# author: Qling
# date: 2021-04-09
# difficulty: Medium
# categories: Array, Binary Search
# link: <https://leetcode.com/problems/search-in-rotated-sorted-array-ii/description/>
# hidden: true
# ---
#
# Suppose an array sorted in ascending order is rotated at some pivot unknown to
# you beforehand.
#
# (i.e., `[0,0,1,2,2,5,6]` might become `[2,5,6,0,0,1,2]`).
#
# You are given a target value to search. If found in the array return `true`,
# otherwise return `false`.
#
# **Example 1:**
#
#
#
#     Input: nums = [2,5,6,0,0,1,2], target = 0
#     Output: true
#
#
# **Example 2:**
#
#
#
#     Input: nums = [2,5,6,0,0,1,2], target = 3
#     Output: false
#
# **Follow up:**
#
#   * This is a follow up problem to [Search in Rotated Sorted Array](https://leetcode.com/problems/search-in-rotated-sorted-array/description/), where `nums` may contain duplicates.
#   * Would this affect the run-time complexity? How and why?
#
#
## @lc code=start
using LeetCode

function search_rotate(nums::Vector{Int}, target::Int)::Bool
    left, right = 1, length(nums)
    while left <= right
        mid = (left + right) >> 1
        (nums[mid] == target) && return true
        if nums[mid] == nums[left]
            left += 1
        elseif nums[mid] <= nums[right]
            if nums[mid] < target <= nums[right]
                left = mid + 1
            else
                right = mid - 1
            end
        else
            if nums[left] <= target < nums[mid]
                right = mid - 1
            else
                left = mid + 1
            end
        end
    end

    return false
end
## @lc code=end


text:
import Knet.Ops20: dropout!, dropback!
using Random: rand!
using CUDA: CUDA, CuArray, CuPtr, seed!
using Knet.KnetArrays: KnetArray
using Knet.LibKnet8: @knet8

for (R,P) in ((CuArray,CuPtr),(KnetArray,Ptr)), T in (Float16, Float32, Float64)
    S = sizeof(T)*8
    forw = Symbol("dropout_$S")
    back = Symbol("dropback_$S")
    @eval begin
        function dropout!(p::Number, x::$R{$T}, y::$R{$T}; seed=0)
            if seed !== 0; CUDA.seed!(seed); end
            rand!(y)
            @knet8($forw,(Cint,$T,$P{$T},$P{$T}),length(y),$T(p),x,y)
            return y
        end
        function dropback!(p::Number, x::$R{$T}, y::$R{$T}, dy::$R{$T}, dx::$R{$T})
            @knet8($back,(Cint,$T,$P{$T},$P{$T},$P{$T},$P{$T}),length(dx),$T(p),x,y,dy,dx)
            return dx
        end
    end
end


text:
module MySQLClient

import Sockets
import SHA

const CLIENT_PROTOCOL_41 = 0x0200
const CLIENT_SECURE_CONNECTION = 0x8000
const CLIENT_LONG_PASSWORD = 0x0004
const CLIENT_TRANSACTIONS = 0x2000
const CLIENT_LONG_FLAG = 0x0004
const CLIENT_PLUGIN_AUTH = 0x00080000

const Byte = UInt8

struct MySQLPacket
    payload_payload_length::Int
    sequence_id::Int
    payload::Vector{Byte}

    function MySQLPacket(header, payload)
        new(
            _little_endian_int(header[1:3]),
            Int(header[4]),
            payload
        )
    end
end
MySQLPacket(sock) = MySQLPacket(_read_packet(sock)...)

struct MySQLConnection end

function connect(host, username, password, port=3306)
    sock = Sockets.connect(host, port)

    function handshake()
        packet = MySQLPacket(sock)
        payload = IOBuffer(copy(packet.payload))

        # https://dev.mysql.com/doc/internals/en/connection-phase-packets.html
        protocol_version = Int(read(payload, 1)[])

        mysql_server_version = String(readuntil(payload, 0x0))
        connection_id = _little_endian_int(read(payload, 4))
        auth_plugin_data_part1 = read(payload, 8)
        filter = read(payload, 1)
        capability_lower = read(payload, 2)
        charset = read(payload, 1)
        status = _little_endian_int(read(payload, 2))
        capability_upper = read(payload, 2)
        capability = reinterpret(UInt32, vcat(capability_lower, capability_upper))[]

        if capability & CLIENT_PLUGIN_AUTH > 0
            len_auth_plugin_data = Int(read(payload, 1)[])
            skip(payload, 10) # reserved

            if capability & CLIENT_SECURE_CONNECTION > 0
                auth_plugin_data_part2 = read(payload, max(13, len_auth_plugin_data - 8))
            end
            auth_plugin_name = String(readuntil(payload, 0x0))
        end

        # hankshake response packet
        #
        ## https://dev.mysql.com/doc/internals/en/secure-password-authentication.html#packet-Authentication::Native41
        # sha1(password) xor concat("20-bytes random data from server", sha1(sha1(password)))
        random_data = vcat(auth_plugin_data_part1, auth_plugin_data_part2)[1:20]
        auth_response = SHA.sha1(SHA.sha1(password))
        auth_response = vcat(random_data, auth_response)
        auth_response = SHA.sha1(auth_response)
        auth_response = xor.(SHA.sha1(password), auth_response)

        # header, capability flags, max-packet size, charset, reserved
        buffer_size = 4 + 4 + 4 + 1 + 23
        buffer_size += length(username) + 1 # 1 = null termination
        buffer_size += 1 # length of auth-response
        buffer_size += length(auth_response)
        buffer_size += length(auth_plugin_name) + 1 # 1 = null termination
        buffer = IOBuffer(maxsize=buffer_size)
        payload_size = buffer_size - 4

        # header
        write(buffer, reinterpret(Byte, [payload_size])[1:3])
        write(buffer, 0x01) # sequence_id 決め打ち

        # payload
        capability_flags = Int32(CLIENT_PROTOCOL_41 | CLIENT_SECURE_CONNECTION
                                  | CLIENT_LONG_PASSWORD | CLIENT_TRANSACTIONS
                                  | CLIENT_LONG_FLAG)
        write(buffer, reinterpret(Byte, [capability_flags])[1:4])
        write(buffer, zeros(Byte, 4)) # max packet size は 0 でいいので skip
        write(buffer, charset)
        write(buffer, zeros(Byte, 23)) # reserved
        write(buffer, transcode(Byte, username))
        write(buffer, 0x0)
        write(buffer, Byte(length(auth_response)))
        write(buffer, auth_response)
        write(buffer, transcode(Byte, auth_plugin_name))
        write(buffer, 0x0)

        write(sock, buffer.data)

        header, payload = _read_packet(sock)
        if (iszero(payload[1]))
            println("Welcome to the MySQL monitor")
            println("Your MySQL connection id is $(connection_id)")
            println("Server version: $(mysql_server_version)")
        else
            println("Connection failed")
        end
    end
    handshake()

    # return MySQLConnection(sock)
    return sock
end

function execute(sock, query)
    write(sock, _com_query(query))

    packet = MySQLPacket(sock)

    payload = copy(packet.payload)
    if payload[1] == 0x00
        println("OK")
        return
    end
    column_count, _ = _read_lenenc_int(payload)

    col_names = String[]
    while (true)
        packet = MySQLPacket(sock)
        payload = copy(packet.payload)
        if payload[1] == 0xfe
            break
        elseif payload[1] == 0x00
            error("Unsupported query result")
        else
            offset = 1
            catalog, len = _read_lenenc_str(payload[offset:end])
            offset += len
            schema, len = _read_lenenc_str(payload[offset:end])
            offset += len
            table, len = _read_lenenc_str(payload[offset:end])
            offset += len
            org_table, len = _read_lenenc_str(payload[offset:end])
            offset += len
            col_name, len = _read_lenenc_str(payload[offset:end])
            push!(col_names, col_name)
            offset += len
            org_name, len = _read_lenenc_str(payload[offset:end])
            offset += len
            offset += 1 # length of fixed length fields
            charset = _little_endian_int(payload[offset:offset+1])
            offset += 2
            column_length = _little_endian_int(payload[offset:offset+3])
            offset += 4
            column_type = payload[offset]
            offset += 1
            flags = _little_endian_int(payload[offset:offset+1])
            offset += 2
            decimals = payload[offset]
            offset += 1
        end
    end

    for col in col_names
        print("\t$(col)")
    end
    println()
    println("-"^30)

    while (true)
        packet = MySQLPacket(sock)
        payload = copy(packet.payload)
        if payload[1] == 0xfe
            break
        else
            offset = 1
            for i in 1:column_count
                value, len = _read_lenenc_str(payload[offset:end])
                offset += len
                print("\t$(value)")
            end
            println()
        end
    end
end

function query(conn::MySQLConnection) end

function _read_packet(sock)
    header = read(sock, 4)
    payload_size = _little_endian_int(header[1:3])
    payload = read(sock, payload_size)

    return header, payload
end

function _little_endian_int(arr::Vector{Byte})
    num = Int64(0)
    for (i, c) in enumerate(arr)
        num += Int64(c) << (8 * (i - 1))
    end
    return num
end

# https://dev.mysql.com/doc/internals/en/com-query.html
function _com_query(query)
    payload_size = length(query) + 1 # 1 = for command_id
    buffer = IOBuffer(maxsize=payload_size+4)
    write(buffer, reinterpret(UInt8, [payload_size])[1:3])
    write(buffer, 0x00) # sequence_id
    write(buffer, 0x03) # command_id
    write(buffer, transcode(UInt8, query))

    return buffer.data
end

function _read_lenenc_int(payload::Vector{Byte})
    io = IOBuffer(copy(payload))
    c = read(io, 1)[]
    offset = 1
    if c == 0xfb
        println("ERROR")
        return 0, offset
    elseif c == 0xfc
        result = _little_endian_int(read(io, 2))
        offset = 3
    elseif c == 0xfd
        result = _little_endian_int(read(io. 3))
        offset = 4
    elseif c == 0xfe
        result = _little_endian_int(read(io. 8))
        offset = 9
    elseif c == 0xff
        println("ERROR PACKET")
    else
        result = Int64(c)
    end

    return result, offset
end

function _read_lenenc_str(payload::Vector{Byte})
    payload = copy(payload)
    len, offset = _read_lenenc_int(payload)
    offset += 1

    result = payload[offset:offset+len-1]

    return String(result), offset + len - 1
end

end # module