text:
# a Row "view" type for iterating `CSV.File`
struct Row <: Tables.AbstractRow
    names::Vector{Symbol}
    columns::Vector{Column}
    lookup::Dict{Symbol, Column}
    row::Int
end

getnames(r::Row) = getfield(r, :names)
getcolumn(r::Row, col::Int) = getfield(r, :columns)[col].column
getcolumn(r::Row, col::Symbol) = getfield(r, :lookup)[col].column
getrow(r::Row) = getfield(r, :row)

Tables.columnnames(r::Row) = getnames(r)

@inline function Tables.getcolumn(row::Row, ::Type{T}, i::Int, nm::Symbol) where {T}
    column = getcolumn(row, i)
    @inbounds x = column[getrow(row)]
    return x
end

@inline function Tables.getcolumn(row::Row, col::Symbol)
    column = getcolumn(row, col)
    @inbounds x = column[getrow(row)]
    return x
end

@inline function Tables.getcolumn(row::Row, col::Int)
    column = getcolumn(row, col)
    @inbounds x = column[getrow(row)]
    return x
end

"""
    CSV.File(input; kwargs...) => CSV.File

Read a UTF-8 CSV input and return a `CSV.File` object, which is like a lightweight table/dataframe, allowing dot-access to columns
and iterating rows. Satisfies the Tables.jl interface, so can be passed to any valid sink, yet to avoid unnecessary copies of data,
use `CSV.read(input, sink; kwargs...)` instead if the `CSV.File` intermediate object isn't needed.

The [`input`](@ref input) argument can be one of:
  * filename given as a string or FilePaths.jl type
  * a `Vector{UInt8}` or `SubArray{UInt8, 1, Vector{UInt8}}` byte buffer
  * a `CodeUnits` object, which wraps a `String`, like `codeunits(str)`
  * a csv-formatted string can also be passed like `IOBuffer(str)`
  * a `Cmd` or other `IO`
  * a gzipped file (or gzipped data in any of the above), which will automatically be decompressed for parsing
  * a `Vector` of any of the above, which will parse and vertically concatenate each source, returning a single, "long" `CSV.File`

To read a csv file from a url, use the Downloads.jl stdlib or HTTP.jl package, where the resulting downloaded tempfile or `HTTP.Response` body can be passed like:
```julia
using Downloads, CSV
f = CSV.File(Downloads.download(url))

# or

using HTTP, CSV
f = CSV.File(HTTP.get(url).body)
```

Opens the file or files and uses passed arguments to detect the number of columns and column types, unless column types are provided
manually via the `types` keyword argument. Note that passing column types manually can slightly increase performance
for each column type provided (column types can be given as a `Vector` for all columns, or specified per column via
name or index in a `Dict`).

When a `Vector` of inputs is provided, the column names and types of each separate file/input must match to be vertically concatenated. Separate threads will
be used to parse each input, which will each parse their input using just the single thread. The results of all threads are then vertically concatenated using
`ChainedVector`s to lazily concatenate each thread's columns.

For text encodings other than UTF-8, load the [StringEncodings.jl](https://github.com/JuliaStrings/StringEncodings.jl)
package and call e.g. `CSV.File(open(read, input, enc"ISO-8859-1"))`.

The returned `CSV.File` object supports the [Tables.jl](https://github.com/JuliaData/Tables.jl) interface
and can iterate `CSV.Row`s. `CSV.Row` supports `propertynames` and `getproperty` to access individual row values. `CSV.File`
also supports entire column access like a `DataFrame` via direct property access on the file object, like `f = CSV.File(file); f.col1`.
Or by getindex access with column names, like `f[:col1]` or `f["col1"]`. The returned columns are `AbstractArray` subtypes, including:
`SentinelVector` (for integers), regular `Vector`, `PooledVector` for pooled columns, `MissingVector` for columns of all `missing` values,
`PosLenStringVector` when `stringtype=PosLenString` is passed, and `ChainedVector` will chain one of the previous array types together for
data inputs that use multiple threads to parse (each thread parses a single "chain" of the input).
Note that duplicate column names will be detected and adjusted to ensure uniqueness (duplicate column name `a` will become `a_1`).
For example, one could iterate over a csv file with column names `a`, `b`, and `c` by doing:

```julia
for row in CSV.File(file)
    println("a=\$(row.a), b=\$(row.b), c=\$(row.c)")
end
```

By supporting the Tables.jl interface, a `CSV.File` can also be a table input to any other table sink function. Like:

```julia
# materialize a csv file as a DataFrame, copying columns from CSV.File
df = CSV.File(file) |> DataFrame

# to avoid making a copy of parsed columns, use CSV.read
df = CSV.read(file, DataFrame)

# load a csv file directly into an sqlite database table
db = SQLite.DB()
tbl = CSV.File(file) |> SQLite.load!(db, "sqlite_table")
```

$KEYWORD_DOCS
"""
struct File <: AbstractVector{Row}
    name::String
    names::Vector{Symbol}
    types::Vector{Type}
    rows::Int
    cols::Int
    columns::Vector{Column}
    lookup::Dict{Symbol, Column}
end

getname(f::File) = getfield(f, :name)
getnames(f::File) = getfield(f, :names)
gettypes(f::File) = getfield(f, :types)
getrows(f::File) = getfield(f, :rows)
getcols(f::File) = getfield(f, :cols)
getcolumns(f::File) = getfield(f, :columns)
getlookup(f::File) = getfield(f, :lookup)
getcolumn(f::File, col::Int) = getfield(f, :columns)[col]
getcolumn(f::File, col::Symbol) = getfield(f, :lookup)[col]

function Base.show(io::IO, f::File)
    println(io, "CSV.File(\"$(getname(f))\"):")
    println(io, "Size: $(getrows(f)) x $(getcols(f))")
    show(io, Tables.schema(f))
end

Base.IndexStyle(::Type{File}) = Base.IndexLinear()
Base.eltype(::File) = Row
Base.size(f::File) = (getrows(f),)

Tables.isrowtable(::Type{File}) = true
Tables.columnaccess(::Type{File}) = true
Tables.schema(f::File)  = Tables.Schema(getnames(f), gettypes(f))
Tables.columns(f::File) = f
Tables.columnnames(f::File) = getnames(f)
Base.propertynames(f::File) = getnames(f)

function Base.getproperty(f::File, col::Symbol)
    lookup = getfield(f, :lookup)
    return haskey(lookup, col) ? lookup[col].column : getfield(f, col)
end

function Base.getindex(f::File, col::Symbol)
    lookup = getfield(f, :lookup)
    return haskey(lookup, col) ? lookup[col].column : getfield(f, col)
end

Base.getindex(f::File, col::String) = getindex(f, Symbol(col))

Tables.getcolumn(f::File, nm::Symbol) = getcolumn(f, nm).column
Tables.getcolumn(f::File, i::Int) = getcolumn(f, i).column

Base.@propagate_inbounds function Base.getindex(f::File, row::Int)
    @boundscheck checkbounds(f, row)
    return Row(getnames(f), getcolumns(f), getlookup(f), row)
end

function File(source::ValidSources;
    # file options
    # header can be a row number, range of rows, or actual string vector
    header::Union{Integer, Vector{Symbol}, Vector{String}, AbstractVector{<:Integer}}=1,
    normalizenames::Bool=false,
    # by default, data starts immediately after header or start of file
    datarow::Integer=-1,
    skipto::Integer=-1,
    footerskip::Integer=0,
    transpose::Bool=false,
    comment::Union{String, Nothing}=nothing,
    ignoreemptyrows::Bool=true,
    ignoreemptylines=nothing,
    select=nothing,
    drop=nothing,
    limit::Union{Integer, Nothing}=nothing,
    buffer_in_memory::Bool=false,
    threaded::Union{Bool, Nothing}=nothing,
    ntasks::Union{Nothing, Integer}=nothing,
    tasks::Union{Nothing, Integer}=nothing,
    rows_to_check::Integer=DEFAULT_ROWS_TO_CHECK,
    lines_to_check=nothing,
    # parsing options
    missingstrings=String[],
    missingstring="",
    delim::Union{Nothing, Char, String}=nothing,
    ignorerepeated::Bool=false,
    quoted::Bool=true,
    quotechar::Union{UInt8, Char}='"',
    openquotechar::Union{UInt8, Char, Nothing}=nothing,
    closequotechar::Union{UInt8, Char, Nothing}=nothing,
    escapechar::Union{UInt8, Char}='"',
    dateformat::Union{String, Dates.DateFormat, Nothing, AbstractDict}=nothing,
    dateformats=nothing,
    decimal::Union{UInt8, Char}=UInt8('.'),
    truestrings::Union{Vector{String}, Nothing}=TRUE_STRINGS,
    falsestrings::Union{Vector{String}, Nothing}=FALSE_STRINGS,
    stripwhitespace::Bool=false,
    # type options
    type=nothing,
    types=nothing,
    typemap::Dict=Dict{Type, Type}(),
    pool=DEFAULT_POOL,
    downcast::Bool=false,
    lazystrings::Bool=false,
    stringtype::StringTypes=DEFAULT_STRINGTYPE,
    strict::Bool=false,
    silencewarnings::Bool=false,
    maxwarnings::Int=DEFAULT_MAX_WARNINGS,
    debug::Bool=false,
    parsingdebug::Bool=false,
    validate::Bool=true,
    )
    # header=1;normalizenames=false;datarow=-1;skipto=-1;footerskip=0;transpose=false;comment=nothing;ignoreemptyrows=true;ignoreemptylines=nothing;
    # select=nothing;drop=nothing;limit=nothing;threaded=nothing;ntasks=Threads.nthreads();tasks=nothing;rows_to_check=30;lines_to_check=nothing;missingstrings=String[];missingstring="";
    # delim=nothing;ignorerepeated=false;quoted=true;quotechar='"';openquotechar=nothing;closequotechar=nothing;escapechar='"';dateformat=nothing;
    # dateformats=nothing;decimal=UInt8('.');truestrings=nothing;falsestrings=nothing;type=nothing;types=nothing;typemap=Dict{Type,Type}();
    # pool=CSV.DEFAULT_POOL;downcast=false;lazystrings=false;stringtype=String;strict=false;silencewarnings=false;maxwarnings=100;debug=false;parsingdebug=false;buffer_in_memory=false
    # @descend CSV.Context(CSV.Arg(source), CSV.Arg(header), CSV.Arg(normalizenames), CSV.Arg(datarow), CSV.Arg(skipto), CSV.Arg(footerskip), CSV.Arg(transpose), CSV.Arg(comment), CSV.Arg(ignoreemptyrows), CSV.Arg(ignoreemptylines), CSV.Arg(select), CSV.Arg(drop), CSV.Arg(limit), CSV.Arg(buffer_in_memory), CSV.Arg(threaded), CSV.Arg(ntasks), CSV.Arg(tasks), CSV.Arg(rows_to_check), CSV.Arg(lines_to_check), CSV.Arg(missingstrings), CSV.Arg(missingstring), CSV.Arg(delim), CSV.Arg(ignorerepeated), CSV.Arg(quoted), CSV.Arg(quotechar), CSV.Arg(openquotechar), CSV.Arg(closequotechar), CSV.Arg(escapechar), CSV.Arg(dateformat), CSV.Arg(dateformats), CSV.Arg(decimal), CSV.Arg(truestrings), CSV.Arg(falsestrings), CSV.Arg(type), CSV.Arg(types), CSV.Arg(typemap), CSV.Arg(pool), CSV.Arg(downcast), CSV.Arg(lazystrings), CSV.Arg(stringtype), CSV.Arg(strict), CSV.Arg(silencewarnings), CSV.Arg(maxwarnings), CSV.Arg(debug), CSV.Arg(parsingdebug), CSV.Arg(false))
    ctx = @refargs Context(source, header, normalizenames, datarow, skipto, footerskip, transpose, comment, ignoreemptyrows, ignoreemptylines, select, drop, limit, buffer_in_memory, threaded, ntasks, tasks, rows_to_check, lines_to_check, missingstrings, missingstring, delim, ignorerepeated, quoted, quotechar, openquotechar, closequotechar, escapechar, dateformat, dateformats, decimal, truestrings, falsestrings, stripwhitespace, type, types, typemap, pool, downcast, lazystrings, stringtype, strict, silencewarnings, maxwarnings, debug, parsingdebug, validate, false)
    return File(ctx)
end

function File(ctx::Context, @nospecialize(chunking::Bool=false))
    @inbounds begin
    # we now do our parsing pass over the file, starting at datapos
    if ctx.threaded
        # multithreaded parsing
        rowsguess, ntasks, columns = ctx.rowsguess, ctx.ntasks, ctx.columns
        # calculate our guess for how many rows will be parsed by each concurrent parsing task
        rowchunkguess = cld(rowsguess, ntasks)
        wholecolumnslock = ReentrantLock() # in case columns are widened during parsing
        pertaskcolumns = Vector{Vector{Column}}(undef, ntasks)
        # initialize each top-level column's lock; used after a task is done parsing its chunk of rows
        # and it "checks in" the types it parsed for each column
        foreach(col -> col.lock = ReentrantLock(), columns)
        rows = zeros(Int, ntasks) # how many rows each parsing task ended up actually parsing
        @sync for i = 1:ntasks
            Threads.@spawn multithreadparse(ctx, pertaskcolumns, rowchunkguess, i, rows, wholecolumnslock)
            # CSV.multithreadparse(ctx, pertaskcolumns, rowchunkguess, i, rows, wholecolumnslock)
        end
        finalrows = sum(rows)
        if ctx.limit < finalrows
            finalrows = ctx.limit
            # adjust columns according to limit
            acc = 0
            for i = 1:ntasks
                if acc + rows[i] > finalrows
                    # need to resize this tasks columns down
                    if finalrows - acc > 0
                        for col in pertaskcolumns[i]
                            if isdefined(col, :column)
                                resize!(col.column, finalrows - acc)
                            end
                        end
                    else
                        for col in pertaskcolumns[i]
                            if isdefined(col, :column)
                                empty!(col.column)
                            end
                        end
                    end
                end
                acc += rows[i]
            end
        end
        # ok, all the parsing tasks have finished and we've promoted their types w/ the top-level columns
        # so now we just need to finish processing each column by making ChainedVectors of the individual columns
        # from each task
        # quick check that each set of task columns has the right # of columns
        for i = 1:ntasks
            task_columns = pertaskcolumns[i]
            if length(task_columns) < length(columns)
                # some other task widened columns that this task didn't likewise detect
                for _ = (length(task_columns) + 1):length(columns)
                    push!(task_columns, Column(Missing, ctx.options))
                end
            end
        end
        @sync for (j, col) in enumerate(columns)
            let finalrows=finalrows
                Threads.@spawn multithreadpostparse(ctx, ntasks, pertaskcolumns, rows, finalrows, j, col)
            end
        end
    else
        # single-threaded parsing
        columns = ctx.columns
        allocate!(columns, ctx.rowsguess)
        t = Base.time()
        finalrows, pos = parsefilechunk!(ctx, ctx.datapos, ctx.len, ctx.rowsguess, 0, columns, ctx.customtypes)::Tuple{Int, Int}
        ctx.debug && println("time for initial parsing: $(Base.time() - t)")
        # cleanup our columns if needed
        for col in columns
@label processcolumn
            if col.type === NeedsTypeDetection
                # fill in uninitialized column fields
                col.type = Missing
                col.column = MissingVector(finalrows)
                col.pool = 0.0
            end
            T = col.anymissing ? Union{col.type, Missing} : col.type
            if maybepooled(col) &&
                (col.type isa StringTypes || col.columnspecificpool) &&
                checkpooled!(T, nothing, col, 0, 1, finalrows, ctx)
                # col.column is a PooledArray
            elseif col.type === PosLenString
                # string col parsed lazily; return a PosLenStringVector
                makeposlen!(col, coltype(col), ctx)
            elseif !col.anymissing
                # if no missing values were parsed for a col, we want to "unwrap" it to a plain Vector{T}
                if col.type === Bool
                    col.column = convert(Vector{Bool}, col.column)
                elseif col.type !== Union{} && col.type <: SmallIntegers
                    col.column = convert(Vector{col.type}, col.column)
                else
                    col.column = parent(col.column)
                end
            end
        end
    end
    # delete any dropped columns from names, columns
    names = ctx.names
    if length(columns) > length(names)
        # columns were widened during parsing, auto-generate trailing column names
        names = makeunique(append!(names, [Symbol(:Column, i) for i = (length(names) + 1):length(columns)]))
    end
    for i = length(columns):-1:1
        col = columns[i]
        if col.willdrop
            deleteat!(names, i)
            deleteat!(columns, i)
        end
    end
    types = Type[coltype(col) for col in columns]
    lookup = Dict(k => v for (k, v) in zip(names, columns))
    ctx.debug && println("types after parsing: $types, pool = $(ctx.pool)")
    # for windows, it's particularly finicky about throwing errors when you try to modify an mmapped file
    # so we just want to make sure we finalize the input buffer so users don't run into surprises
    if !chunking && Sys.iswindows() && ctx.stringtype !== PosLenString
        finalize(ctx.buf)
    end
    # check if a temp file was generated for parsing
    if !chunking && ctx.tempfile !== nothing && ctx.stringtype !== PosLenString
        rm(ctx.tempfile; force=true)
    end
    end # @inbounds begin
    return File(ctx.name, names, types, finalrows, length(columns), columns, lookup)
end

function multithreadparse(ctx, pertaskcolumns, rowchunkguess, i, rows, wholecolumnslock)
    columns = ctx.columns
    tt = Base.time()
    task_columns = [Column(col) for col in columns] # task-local columns derived from top-level columns
    allocate!(task_columns, rowchunkguess)
    pertaskcolumns[i] = task_columns
    task_pos = ctx.chunkpositions[i]
    task_len = ctx.chunkpositions[i + 1] - (i != ctx.ntasks)
    # for error-reporting purposes, we want to try and give the best guess of where a row emits a warning/error, so compute that
    rowchunkoffset = (ctx.datarow - 1) + (rowchunkguess * (i - 1))
    task_rows, task_pos = parsefilechunk!(ctx, task_pos, task_len, rowchunkguess, rowchunkoffset, task_columns, ctx.customtypes)::Tuple{Int, Int}
    rows[i] = task_rows
    # promote column types/flags this task detected while parsing
    lock(wholecolumnslock) do
        # check if this task widened columns while parsing
        if length(task_columns) > length(columns)
            for j = (length(columns) + 1):length(task_columns)
                col = task_columns[j] # I'm pretty sure it's ok to just use the per-task column directly here as new top-level column?
                # initialize lock since it hasn't been initialized yet
                col.lock = ReentrantLock()
                push!(columns, col)
            end
        end
    end
    # now we know that columns is at least as long as task_columns
    for j = 1:length(task_columns)
        col = columns[j]
        # note col.lock is shared amongst all tasks (i.e. belongs to parent columns[i].lock)
        lock(col.lock) do
            task_col = task_columns[j]
            T = col.type
            col.type = something(promote_types(T, task_col.type), ctx.stringtype)
            if T !== col.type
                ctx.debug && println("promoting col = $j from $T to $(col.type), task chunk ($i) was type = $(task_col.type)")
            end
            col.anymissing |= task_col.anymissing
        end
    end
    ctx.debug && println("finished parsing $task_rows rows on task = $i: time for parsing: $(Base.time() - tt)")
    return
end

function multithreadpostparse(ctx, ntasks, pertaskcolumns, rows, finalrows, j, col)
    # first check if need to re-parse any chunks
    for i = 1:ntasks
        task_columns = pertaskcolumns[i]
        task_col = task_columns[j]
        task_rows = rows[i]
        # check if we need to promote a task-local column based on what other threads parsed
        T = col.type # final promoted type from amongst all separate parsing tasks
        T2 = task_col.type
        if T isa StringTypes && !(T2 isa StringTypes)
            # promoting non-string to string column
            ctx.debug && println("multithreaded promoting column $j to string from $T2")
            task_len = ctx.chunkpositions[i + 1] - (i != ntasks)
            task_pos = ctx.chunkpositions[i]
            promotetostring!(ctx, ctx.buf, task_pos, task_len, task_rows, sum(rows[1:i-1]), task_columns, ctx.customtypes, j, Ref(0), task_rows, T)
            col.type = something(promote_types(T, task_columns[j].type), ctx.stringtype)
            T = col.type
        end
    end
    for i = 1:ntasks
        task_columns = pertaskcolumns[i]
        task_col = task_columns[j]
        task_rows = rows[i]
        # check if we need to promote a task-local column based on what other threads parsed
        T = col.type # final promoted type from amongst all separate parsing tasks
        T2 = task_col.type
        if T === Float64 && T2 <: Integer
            # one chunk parsed as Int, another as Float64, promote to Float64
            ctx.debug && println("multithreaded promoting column $j to float")
            task_col.column = convert(SentinelVector{Float64}, task_col.column)
        elseif T !== T2 && (T <: InlineString || (T === String && T2 <: InlineString))
            # promote to widest InlineString type
            task_col.column = convert(SentinelVector{T}, task_col.column)
        elseif T !== T2
            # one chunk parsed all missing values, but another chunk had a typed value, promote to that
            # while keeping all values `missing` (allocate by default ensures columns have all missing values)
            ctx.debug && println("multithreaded promoting column $j from missing on task $i")
            task_col.column = allocate(T, task_rows)
        end
    end
    T = col.anymissing ? Union{col.type, Missing} : col.type
    if maybepooled(col) &&
        (col.type isa StringTypes || col.columnspecificpool) &&
        checkpooled!(T, pertaskcolumns, col, j, ntasks, finalrows, ctx)
        # col.column is a PooledArray
    elseif col.type === Int64
        # we need to special-case Int here because while parsing, a default Int64 sentinel value is chosen to
        # represent missing; if any chunk bumped into that sentinel value while parsing, then it cycled to a
        # new sentinel value; this step ensures that each chunk has the same encoded sentinel value
        # passing force=false means it will first check if all chunks already have the same sentinel and return
        # immediately if so, which will be the case most often
        SentinelArrays.newsentinel!((pertaskcolumns[i][j].column::SVec{Int64} for i = 1:ntasks)...; force=false)
        makechain!(col.type, pertaskcolumns, col, j, ntasks)
    elseif col.type === PosLenString
        col.column = ChainedVector(PosLenStringVector{coltype(col)}[makeposlen!(pertaskcolumns[i][j], coltype(col), ctx) for i = 1:ntasks])
    elseif col.type === NeedsTypeDetection || col.type === HardMissing
        col.type = Missing
        col.column = MissingVector(finalrows)
    else
        makechain!(col.type, pertaskcolumns, col, j, ntasks)
    end
    if finalrows < length(col.column)
        # we only ever resize! down here, so no need to use reallocate!
        resize!(col.column, finalrows)
    end
    return
end

function makechain!(::Type{T}, pertaskcolumns, col, j, ntasks) where {T}
    if col.anymissing
        col.column = ChainedVector([pertaskcolumns[i][j].column for i = 1:ntasks])
    else
        if col.type === Bool
            col.column = ChainedVector([convert(Vector{Bool}, pertaskcolumns[i][j].column::vectype(T)) for i = 1:ntasks])
        elseif col.type !== Union{} && col.type <: SmallIntegers
            col.column = ChainedVector([convert(Vector{col.type}, pertaskcolumns[i][j].column::vectype(T)) for i = 1:ntasks])
        else
            col.column = ChainedVector([parent(pertaskcolumns[i][j].column) for i = 1:ntasks])
        end
    end
    return
end

# T is Union{T, Missing} or T depending on col.anymissing
function checkpooled!(::Type{T}, pertaskcolumns, col, j, ntasks, nrows, ctx) where {T}
    S = Base.nonmissingtype(T)
    pool = Dict{T, UInt32}()
    lastref = Ref{UInt32}(0)
    refs = Vector{UInt32}(undef, nrows)
    k = 1
    limit = col.pool isa Tuple ? col.pool[2] : typemax(Int)
    for i = 1:ntasks
        column = (pertaskcolumns === nothing ? col.column : pertaskcolumns[i][j].column)::columntype(S)
        for x in column
            if x isa PosLen
                if x.missingvalue
                    refs[k] = get!(pool, missing) do
                        lastref[] += UInt32(1)
                    end
                elseif x.escapedvalue
                    val = S === PosLenString ? S(ctx.buf, x, ctx.options.e) : Parsers.getstring(ctx.buf, x, ctx.options.e)
                    refs[k] = get!(pool, val) do
                        lastref[] += UInt32(1)
                    end
                else
                    val = PointerString(pointer(ctx.buf, x.pos), x.len)
                    index = Base.ht_keyindex2!(pool, val)
                    if index > 0
                        found_key = pool.vals[index]
                        ref = found_key::UInt32
                    else
                        new = lastref[] += UInt32(1)
                        if S === PosLenString
                            Base._setindex!(pool, new, S(ctx.buf, x, ctx.options.e), -index)
                        else
                            Base._setindex!(pool, new, S(val), -index)
                        end
                        ref = new
                    end
                    refs[k] = ref
                end
            else
                refs[k] = get!(pool, x) do
                    lastref[] += UInt32(1)
                end
            end
            k += 1
            if length(pool) > limit
                return false
            end
        end
    end
    cpool = col.pool
    percent = cpool isa Tuple ? cpool[1] : cpool
    if ((length(pool) - 1) / nrows) <= percent
        col.column = PooledArray(PooledArrays.RefArray(refs), pool)
        return true
    else
        return false
    end
end

function makeposlen!(col, T, ctx)
    col.column = PosLenStringVector{T}(ctx.buf, col.column::Vector{PosLen}, ctx.options.e)
    return col.column
end

function parsefilechunk!(ctx::Context, pos, len, rowsguess, rowoffset, columns, ::Type{customtypes})::Tuple{Int, Int} where {customtypes}
    buf = ctx.buf
    transpose = ctx.transpose
    limit = ctx.limit
    row = 0
    startpos = pos
    if pos <= len && len > 0 && row < limit
        numwarnings = Ref(0)
        while true
            row += 1
            # @show columns
            @inbounds pos = parserow(startpos, row, numwarnings, ctx, buf, pos, len, rowsguess, rowoffset, columns, customtypes)::Int
            # @show columns
            row == limit && break
            (transpose ? all(c -> c.position >= c.endposition, columns) : pos > len) && break
            # if our initial row estimate was too few, we need to reallocate our columns to read the rest of the file/chunk
            if !transpose && row + 1 > rowsguess
                # (bytes left in file/chunk) / (avg bytes per row) == estimated rows left in file (+ 5% to try and avoid reallocating)
                estimated_rows_left = ceil(Int, ((len - pos) / ((pos - startpos) / row)) * 1.05)
                newrowsguess = rowsguess + estimated_rows_left
                newrowsguess = max(rowsguess + 1, newrowsguess)
                ctx.debug && reallocatecolumns(rowoffset + row, rowsguess, newrowsguess)
                for col in columns
                    isdefined(col, :column) && reallocate!(col.column, newrowsguess)
                end
                rowsguess = newrowsguess
            end
        end
    end
    # done parsing (at least this chunk), so resize columns to final row count
    for col in columns
        # we only ever resize! down here, so no need to use reallocate!
        isdefined(col, :column) && resize!(col.column, row)
    end
    return row, pos
end

@noinline reallocatecolumns(row, old, new) = @warn("thread = $(Threads.threadid()) warning: didn't pre-allocate enough column while parsing around row $row, re-allocating from $old to $new...")
@noinline notenoughcolumns(cols, ncols, row) = @warn("thread = $(Threads.threadid()) warning: only found $cols / $ncols columns around data row: $row. Filling remaining columns with `missing`")
@noinline toomanycolumns(cols, row) = @warn("thread = $(Threads.threadid()) warning: parsed expected $cols columns, but didn't reach end of line around data row: $row. Parsing extra columns and widening final columnset")
@noinline stricterror(T, buf, pos, len, code, row, col) = throw(Error("thread = $(Threads.threadid()) error parsing $T around row = $row, col = $col: \"$(String(buf[pos:pos+len-1]))\", error=$(Parsers.codes(code))"))
@noinline warning(T, buf, pos, len, code, row, col) = @warn("thread = $(Threads.threadid()) warning: error parsing $T around row = $row, col = $col: \"$(String(buf[pos:pos+len-1]))\", error=$(Parsers.codes(code))")
@noinline fatalerror(buf, pos, len, code, row, col) = throw(Error("thread = $(Threads.threadid()) fatal error, encountered an invalidly quoted field while parsing around row = $row, col = $col: \"$(String(buf[pos:pos+len-1]))\", error=$(Parsers.codes(code)), check your `quotechar` arguments or manually fix the field in the file itself"))
@noinline toomanywwarnings() = @warn("thread = $(Threads.threadid()): too many warnings, silencing any further warnings")

Base.@propagate_inbounds function parserow(startpos, row, numwarnings, ctx::Context, buf, pos, len, rowsguess, rowoffset, columns, ::Type{customtypes})::Int where {customtypes}
    # @show columns
    ncols = length(columns)
    for i = 1:ncols
        col = columns[i]
        if ctx.transpose
            pos = col.position
        end
        type = col.type
        cellstartpos = pos
        if type === HardMissing
            pos, code = parsevalue!(Missing, buf, pos, len, row, rowoffset, i, col, ctx)
        elseif type === NeedsTypeDetection
            pos, code = detectcell(buf, pos, len, row, rowoffset, i, col, ctx, rowsguess)
        elseif type === Int8
            pos, code = parsevalue!(Int8, buf, pos, len, row, rowoffset, i, col, ctx)
        elseif type === Int16
            pos, code = parsevalue!(Int16, buf, pos, len, row, rowoffset, i, col, ctx)
        elseif type === Int32
            pos, code = parsevalue!(Int32, buf, pos, len, row, rowoffset, i, col, ctx)
        elseif type === Int64
            pos, code = parsevalue!(Int64, buf, pos, len, row, rowoffset, i, col, ctx)
        elseif type === Int128
            pos, code = parsevalue!(Int128, buf, pos, len, row, rowoffset, i, col, ctx)
        elseif type === Float16
            pos, code = parsevalue!(Float16, buf, pos, len, row, rowoffset, i, col, ctx)
        elseif type === Float32
            pos, code = parsevalue!(Float32, buf, pos, len, row, rowoffset, i, col, ctx)
        elseif type === Float64
            pos, code = parsevalue!(Float64, buf, pos, len, row, rowoffset, i, col, ctx)
        elseif type === InlineString1
            pos, code = parsevalue!(InlineString1, buf, pos, len, row, rowoffset, i, col, ctx)
        elseif type === InlineString3
            pos, code = parsevalue!(InlineString3, buf, pos, len, row, rowoffset, i, col, ctx)
        elseif type === InlineString7
            pos, code = parsevalue!(InlineString7, buf, pos, len, row, rowoffset, i, col, ctx)
        elseif type === InlineString15
            pos, code = parsevalue!(InlineString15, buf, pos, len, row, rowoffset, i, col, ctx)
        elseif type === InlineString31
            pos, code = parsevalue!(InlineString31, buf, pos, len, row, rowoffset, i, col, ctx)
        elseif type === InlineString63
            pos, code = parsevalue!(InlineString63, buf, pos, len, row, rowoffset, i, col, ctx)
        elseif type === InlineString127
            pos, code = parsevalue!(InlineString127, buf, pos, len, row, rowoffset, i, col, ctx)
        elseif type === InlineString255
            pos, code = parsevalue!(InlineString255, buf, pos, len, row, rowoffset, i, col, ctx)
        elseif type === String
            pos, code = parsevalue!(String, buf, pos, len, row, rowoffset, i, col, ctx)
        elseif type === PosLenString
            pos, code = parsevalue!(PosLenString, buf, pos, len, row, rowoffset, i, col, ctx)
        elseif type === Date
            pos, code = parsevalue!(Date, buf, pos, len, row, rowoffset, i, col, ctx)
        elseif type === DateTime
            pos, code = parsevalue!(DateTime, buf, pos, len, row, rowoffset, i, col, ctx)
        elseif type === Time
            pos, code = parsevalue!(Time, buf, pos, len, row, rowoffset, i, col, ctx)
        elseif type === Bool
            pos, code = parsevalue!(Bool, buf, pos, len, row, rowoffset, i, col, ctx)
        else
            if customtypes !== Tuple{}
                pos, code = parsecustom!(customtypes, buf, pos, len, row, rowoffset, i, col, ctx)
            else
                error("bad column type: $(type))")
            end
        end
        if promote_to_string(code)
            ctx.debug && println("promoting column i = $i to string from $(type) on chunk = $(Threads.threadid())")
            if type <: InlineString
                newT = String
            elseif ctx.stringtype === InlineString
                str = Parsers.xparse(String, buf, cellstartpos, len, col.options)
                newT = pickstringtype(InlineString, str.val.len)
            else
                newT = ctx.stringtype
            end
            promotetostring!(ctx, buf, startpos, len, rowsguess, rowoffset, columns, customtypes, i, numwarnings, row, newT)
        end
        if ctx.transpose
            col.position = pos
        else
            if i < ncols
                if Parsers.newline(code) || pos > len
                    # in https://github.com/JuliaData/CSV.jl/issues/948,
                    # it was noticed that if we reached the EOF right before parsing
                    # the last expected column, then the warning is a bit spurious.
                    # The final value is `missing` and the csv writer chose to just
                    # "close" the file w/o including a final newline
                    # we can treat this special-case as "valid" and not emit a warning
                    if !(pos > len && i == (ncols - 1))
                        ctx.silencewarnings || numwarnings[] > ctx.maxwarnings || notenoughcolumns(i, ncols, rowoffset + row)
                        !ctx.silencewarnings && numwarnings[] == ctx.maxwarnings && toomanywwarnings()
                        numwarnings[] += 1
                    end
                    for j = (i + 1):ncols
                        columns[j].anymissing = true
                    end
                    break # from for i = 1:ncols
                end
            elseif pos <= len && !Parsers.newline(code)
                # extra columns on this row, let's widen
                ctx.silencewarnings || toomanycolumns(ncols, rowoffset + row)
                j = i + 1
                T = ctx.streaming ? Union{ctx.stringtype, Missing} : NeedsTypeDetection
                while pos <= len && !Parsers.newline(code)
                    col = Column(T, ctx.options)
                    col.anymissing = ctx.streaming || rowoffset == 0 && row > 1 # assume all previous rows were missing
                    col.pool = ctx.pool
                    if T === NeedsTypeDetection
                        pos, code = detectcell(buf, pos, len, row, rowoffset, j, col, ctx, rowsguess)
                    else
                        # need to allocate
                        col.column = allocate(ctx.stringtype, ctx.rowsguess)
                        pos, code = parsevalue!(ctx.stringtype, buf, pos, len, row, rowoffset, j, col, ctx)
                    end
                    j += 1
                    push!(columns, col)
                end
            end
        end
    end
    return pos
end

function detectcell(buf, pos, len, row, rowoffset, i, col, ctx, rowsguess)::Tuple{Int, Int16}
    # debug && println("detecting on task $(Threads.threadid())")
    opts = col.options
    code, tlen, x, xT = detect(pass, buf, pos, len, opts, false, ctx.downcast, rowoffset + row, i)
    if x === missing
        col.anymissing = true
        @goto finaldone
    end
    newT = ctx.stringtype
    if x !== nothing
        # we found a non-missing value
        newT = get(ctx.typemap, typeof(x), typeof(x))
        if !(newT isa StringTypes)
            if newT !== typeof(x)
                # type-mapping typeof(x) => newT
                # this ultimate call to Parsers.xparse has no hope in inference (because of the typeof(x) => newT mapping)
                # so we "outline" the call and assert the types of everything but `y` to make sure `code` and `tlen` stay type stable
                res = _parseany(newT, buf, pos, len, opts)
                code, tlen = res.code, res.tlen
                if Parsers.ok(code)
                    val = res.val
                    @goto done
                end
            else
                val = x
                @goto done
            end
        end
    end
    # if we "fall through" to here, that means we either detected a string value
    # or we're type-mapping from another detected type to string
    str = Parsers.xparse(String, buf, pos, len, opts)
    poslen = str.val
    if newT === InlineString && poslen.len < DEFAULT_MAX_INLINE_STRING_LENGTH
        newT = InlineStringType(poslen.len)
        val = newT(PosLenString(buf, poslen, opts.e))
    elseif newT === PosLenString
        newT = PosLenString
        val = poslen
    else
        newT = String
        val = Parsers.getstring(buf, poslen, opts.e)
    end
@label done
    # if we're here, that means we found a non-missing value, so we need to update column
    column = allocate(newT, rowsguess)
    column[row] = val
    col.column = column
    col.type = newT
@label finaldone
    return pos + tlen, code
end

function parsevalue!(::Type{type}, buf, pos, len, row, rowoffset, i, col, ctx)::Tuple{Int, Int16} where {type}
    opts = col.options
    res = Parsers.xparse(type === Missing ? String : type, buf, pos, len, opts)
    code = res.code
    if !Parsers.invalid(code)
        if type !== Missing
            if Parsers.sentinel(code)
                col.anymissing = true
            else
                column = col.column
                val = res.val
                if column isa Vector{PosLen} && val isa PosLen
                    @inbounds (column::Vector{PosLen})[row] = val
                elseif type === String
                    @inbounds (column::SVec2{String})[row] = Parsers.getstring(buf, val, opts.e)
                else
                    @inbounds (column::vectype(type))[row] = val
                end
            end
        end
    else
        # something went wrong parsing
        if Parsers.invalidquotedfield(code)
            # this usually means parsing is borked because of an invalidly quoted field, hard error
            fatalerror(buf, pos, res.tlen, code, rowoffset + row, i)
        end
        if type !== Missing && type !== PosLenString && type !== String
            if col.userprovidedtype
                if !ctx.strict
                    ctx.silencewarnings || warning(type, buf, pos, res.tlen, code, rowoffset + row, i)
                    col.anymissing = true
                else
                    stricterror(type, buf, pos, res.tlen, code, rowoffset + row, i)
                end
            else
                if type === Int8 || type === Int16 || type === Int32 || type === Int64 || type === Int128
                    newT = _widen(type)
                    while newT !== nothing && !Parsers.ok(code)
                        newT = get(ctx.typemap, newT, newT)
                        if newT isa StringTypes
                            code |= PROMOTE_TO_STRING
                            break
                        end
                        code = trytopromote!(type, newT, buf, pos, len, col, row)
                        newT = _widen(newT)
                    end
                elseif type === InlineString1 || type === InlineString3 || type === InlineString7 || type === InlineString15
                    newT = widen(type)
                    while newT !== InlineString63
                        ret = _parseany(newT, buf, pos, len, opts)
                        if !Parsers.invalid(ret.code)
                            col.type = newT
                            column = col.column
                            col.column = convert(SentinelVector{newT}, col.column::vectype(type))
                            @inbounds col.column[row] = ret.val
                            return pos + ret.tlen, ret.code
                        end
                        newT = widen(newT)
                    end
                    #TODO: should we just convert(SentinelVector{String}) here?
                    code |= PROMOTE_TO_STRING
                else
                    code |= PROMOTE_TO_STRING
                end
            end
        end
    end
    return pos + res.tlen, code
end

@noinline function trytopromote!(::Type{from}, ::Type{to}, buf, pos, len, col, row)::Int16 where {from, to}
    res = Parsers.xparse(to, buf, pos, len, col.options)
    code = res.code
    if !Parsers.invalid(code)
        col.type = to
        column = col.column
        if column isa vectype(from)
            col.column = convert(promotevectype(to), column)
            @inbounds col.column[row] = res.val
        end
    else
        code |= PROMOTE_TO_STRING
    end
    return code
end

@inline function parsecustom!(::Type{customtypes}, buf, pos, len, row, rowoffset, i, col, ctx) where {customtypes}
    if @generated
        block = Expr(:block)
        push!(block.args, quote
            error("CSV.jl code-generation error, unexpected column type: $(type)")
        end)
        for i = 1:fieldcount(customtypes)
            T = fieldtype(customtypes, i)
            pushfirst!(block.args, quote
                if type === $T
                    return parsevalue!($T, buf, pos, len, row, rowoffset, i, col, ctx)
                end
            end)
        end
        pushfirst!(block.args, :(type = col.type))
        pushfirst!(block.args, Expr(:meta, :inline))
        # @show block
        return block
    else
        # println("generated function failed")
        return parsevalue!(col.type, buf, pos, len, row, rowoffset, i, col, ctx)
    end
end

@noinline function promotetostring!(ctx::Context, buf, pos, len, rowsguess, rowoffset, columns, ::Type{customtypes}, column_to_promote, numwarnings, limit, stringtype) where {customtypes}
    cols = [i == column_to_promote ? columns[i] : Column(Missing, columns[i].options) for i = 1:length(columns)]
    col = cols[column_to_promote]
    col.column = allocate(stringtype, rowsguess)
    col.type = stringtype
    row = 0
    startpos = pos
    if pos <= len && len > 0
        while row < limit
            row += 1
            @inbounds pos = parserow(startpos, row, numwarnings, ctx, buf, pos, len, rowsguess, rowoffset, cols, customtypes)
            pos > len && break
        end
    end
    return
end

function File(sources::Vector;
    source::Union{Nothing, Symbol, AbstractString,
        Pair{<:Union{Symbol, AbstractString}, <:AbstractVector}}=nothing,
    kw...)
    isempty(sources) && throw(ArgumentError("unable to read delimited data from empty sources array"))
    if source isa Pair
        length(source.second) == length(sources) || throw(ArgumentError("source pair keyword argument list ($(length(source.second))) must match length of input vector ($(length(sources)))"))
    end
    length(sources) == 1 && return File(sources[1]; kw...)
    all(x -> x isa ValidSources, sources) || throw(ArgumentError("all provided sources must be one of: `$ValidSources`"))
    kws = merge(values(kw), (ntasks=1,))
    f = File(sources[1]; kws...)
    rows = f.rows
    for col in f.columns
        col.column = ChainedVector([col.column])
    end
    files = Vector{File}(undef, length(sources) - 1)
    @sync for i = 2:length(sources)
        Threads.@spawn begin
            files[i - 1] = File(sources[i]; kws...)
        end
    end
    lookup = f.lookup
    for i = 2:length(sources)
        f2 = files[i - 1]
        rows += f2.rows
        fl2 = f2.lookup
        for (nm, col) in lookup
            if haskey(fl2, nm)
                col.column = chaincolumns!(col.column, fl2[nm].column)
            else
                col.column = chaincolumns!(col.column, MissingVector(f2.rows))
            end
        end
    end
    if source !== nothing
        # add file name of each "partition" as 1st column
        pushfirst!(files, f)
        vals = source isa Pair ? source.second : [f.name for f in files]
        pool = Dict(x => UInt32(i) for (i, x) in enumerate(vals))
        arr = PooledArray(PooledArrays.RefArray(ChainedVector([fill(UInt32(i), f.rows) for (i, f) in enumerate(files)])), pool)
        col = Column(eltype(arr))
        col.column = arr
        push!(f.columns, col)
        colnm = Symbol(source isa Pair ? source.first : source)
        push!(f.names, colnm)
        push!(f.types, eltype(arr))
        f.lookup[colnm] = col
    end
    return File(f.name, f.names, f.types, rows, f.cols, f.columns, f.lookup)
end


text:
 #= The border is drawn on line X but since it has a thickness this must be taken into account!
 ┌──────────────────────────
 |  margin
 |    ┌──────────────────────────
 |    │  border------------------ <──────────────────────────┐
 |    │  :  ┌─────────────────────                           │
 |    │  :  │  padding             }──────────────┐          │
 |    │  :  │    ┌───────────┐ <──────── top - padding - (border * 0.5)
 |    │  :  │    │  content  │
 |    │  :  │    └───────────┘

 left, top, right, bottom = content-area
 ...padding, border and margin are added to that!
 ┌───────────────────────────────┐
 │  margin                       │
 │    ┌─────────────────────┐    │
 │    │  padding            │    │
 │    │    ┌───────────┐    │    │
 │    │    │  content  │    │    │
 │    │    └───────────┘    │    │
 │    │                     │    │
 │    └─border──────────────┘    │
 │                               │
 └───────────────────────────────┘

                                                                              =#

export setDistanceFromBounds, getBorderBox, getContentBox, getMarginBox, getSize,
       TotalShapeWidth, getPackingWidth, TotalShapeHeight,
       DrawANode, InitializeRow, getReal
# ==============================================================================
# To set absolute position
function setDistanceFromBounds(l,t,w,h,  shape )
    shape.flags[Bottom] ? (shape.top = t+h - (shape.height+shape.top)) : (shape.top = t+shape.top)
    shape.flags[Right] ? (shape.left = l+w - (shape.width+shape.left)) : (shape.left = l+shape.left)
end
# ==============================================================================
function getReal(box::Draw)
     return (  get(box.padding, BoxOutline(0,0,0,0,0,0)),
               get(box.border,  Border(0,0,0,0,0,0, 0,[],[0,0,0,0])),
               get(box.margin,  BoxOutline(0,0,0,0,0,0))    )
 end
 # ==============================================================================
function getBorderBox(shape::Draw, border::Border, padding::BoxOutline)
     return  shape.left   - padding.left   - (border.left *.5),
             shape.top    - padding.top    - (border.top *.5),
             shape.width  + padding.width  + (border.width),
             shape.height + padding.height + (border.height);
end

getBorderBox(shape::TextLine) = getContentBox(shape)
 #getBorderBox(shape, get(shape.border, Border()), get(shape.padding, BoxOutline()))

getBorderBox(shape) = getBorderBox(shape, get(shape.border, Border()),
                       get(shape.padding, BoxOutline()))

getContentBox(box) = ( box.left, box.top, box.width, box.height )
# ==============================================================================
contentOffset(padding::BoxOutline, border::Border, margin::BoxOutline) =
   (padding.left + border.left + margin.left) , (padding.top + border.top + margin.top)

contentOffset(shape::TextLine) = (0,0)

macro ContentLeft(pl, bl, ml)
    return :( $(pl) + $(bl) + $(ml) )
end
macro ContentTop(pt, bt, mt)
    return :( $(pt) + $(bt) + $(mt) )
end

function contentOffset(shape::Draw)
    padding, border ,margin = getReal(shape)
      return (padding.left + border.left + margin.left) , (padding.top + border.top + margin.top)
  end
# ==============================================================================
function getSize(box::NBox)
     padding, border ,margin = getReal(box)
       return ( box.width  + border.width  + margin.width  + padding.width,
                box.height + border.height + margin.height + padding.height )
end
function getWidthfromContentLeft(box)
     padding, border ,margin = getReal(box)
       return ( box.width  + border.right  + margin.right  + padding.right )
end
getSize(text::TextLine) = ( text.width, text.height )

function getPackingWidth(box) # should be getOutersWidth
     padding, border ,margin = getReal(box)
       return ( box.width  + border.width  + margin.width  + padding.width )
end

function getSize(circle::Circle)
     border = get(circle.border,  Border(0,0,0,0,0,0, 0,[],[0,0,0,0]))
     margin = get(circle.margin,  BoxOutline(0,0,0,0,0,0))
       dia = circle.radius*2+1
       return ( dia + border.width  + margin.width ,
                dia + border.height + margin.height )
end

macro BorderLeft(box::Draw, border::Border, padding::BoxOutline)
    return :( $(box).left  - $(padding).left  - ($(border).left *.5))
end
macro BorderTop(box::Draw, border::Border, padding::BoxOutline)
    return :( $(box).top  - $(padding).top  - ($(border).top *.5))
end
macro BorderRight(box::Draw, border::Border, padding::BoxOutline)
    return :( $(box).width  + $(padding).width  + ($(border).width))
end
macro BorderBottom(box::Draw, border::Border, padding::BoxOutline)
    return :( $(box).height  + $(padding).height  + ($(border).height))
end

using Colors
color_names = Colors.color_names
include("GraphDraw.jl")
include("GraphText.jl")
include("GraphFlags.jl")

ext: jl


text:
#=
This file is auto-generated. Do not edit.
=#
"""
    mutable struct AGC <: Service
        name::String
        available::Bool
        bias::Float64
        K_p::Float64
        K_i::Float64
        K_d::Float64
        delta_t::Float64
        area::Union{Nothing, Area}
        initial_ace::Float64
        ext::Dict{String, Any}
        internal::InfrastructureSystemsInternal
    end

# Arguments
- `name::String`
- `available::Bool`
- `bias::Float64`
- `K_p::Float64`: PID Proportional Constant
- `K_i::Float64`: PID Integral Constant
- `K_d::Float64`: PID Derrivative Constant
- `delta_t::Float64`: PID Discretization period [Seconds]
- `area::Union{Nothing, Area}`: the area controlled by the AGC
- `initial_ace::Float64`: PID Discretization period [Seconds]
- `ext::Dict{String, Any}`
- `internal::InfrastructureSystemsInternal`: power system internal reference, do not modify
"""
mutable struct AGC <: Service
    name::String
    available::Bool
    bias::Float64
    "PID Proportional Constant"
    K_p::Float64
    "PID Integral Constant"
    K_i::Float64
    "PID Derrivative Constant"
    K_d::Float64
    "PID Discretization period [Seconds]"
    delta_t::Float64
    "the area controlled by the AGC"
    area::Union{Nothing, Area}
    "PID Discretization period [Seconds]"
    initial_ace::Float64
    ext::Dict{String, Any}
    "power system internal reference, do not modify"
    internal::InfrastructureSystemsInternal
end

function AGC(name, available, bias, K_p, K_i, K_d, delta_t, area=nothing, initial_ace=0.0, ext=Dict{String, Any}(), )
    AGC(name, available, bias, K_p, K_i, K_d, delta_t, area, initial_ace, ext, InfrastructureSystemsInternal(), )
end

function AGC(; name, available, bias, K_p, K_i, K_d, delta_t, area=nothing, initial_ace=0.0, ext=Dict{String, Any}(), )
    AGC(name, available, bias, K_p, K_i, K_d, delta_t, area, initial_ace, ext, )
end

# Constructor for demo purposes; non-functional.
function AGC(::Nothing)
    AGC(;
        name="init",
        available=false,
        bias=0.0,
        K_p=0.0,
        K_i=0.0,
        K_d=0.0,
        delta_t=0.0,
        area=Area(nothing),
        initial_ace=0.0,
        ext=Dict{String, Any}(),
    )
end

InfrastructureSystems.get_name(value::AGC) = value.name
"""Get AGC available."""
get_available(value::AGC) = value.available
"""Get AGC bias."""
get_bias(value::AGC) = value.bias
"""Get AGC K_p."""
get_K_p(value::AGC) = value.K_p
"""Get AGC K_i."""
get_K_i(value::AGC) = value.K_i
"""Get AGC K_d."""
get_K_d(value::AGC) = value.K_d
"""Get AGC delta_t."""
get_delta_t(value::AGC) = value.delta_t
"""Get AGC area."""
get_area(value::AGC) = value.area
"""Get AGC initial_ace."""
get_initial_ace(value::AGC) = value.initial_ace
"""Get AGC ext."""
get_ext(value::AGC) = value.ext
"""Get AGC internal."""
get_internal(value::AGC) = value.internal

InfrastructureSystems.set_name!(value::AGC, val::String) = value.name = val
"""Set AGC available."""
set_available!(value::AGC, val::Bool) = value.available = val
"""Set AGC bias."""
set_bias!(value::AGC, val::Float64) = value.bias = val
"""Set AGC K_p."""
set_K_p!(value::AGC, val::Float64) = value.K_p = val
"""Set AGC K_i."""
set_K_i!(value::AGC, val::Float64) = value.K_i = val
"""Set AGC K_d."""
set_K_d!(value::AGC, val::Float64) = value.K_d = val
"""Set AGC delta_t."""
set_delta_t!(value::AGC, val::Float64) = value.delta_t = val
"""Set AGC area."""
set_area!(value::AGC, val::Union{Nothing, Area}) = value.area = val
"""Set AGC initial_ace."""
set_initial_ace!(value::AGC, val::Float64) = value.initial_ace = val
"""Set AGC ext."""
set_ext!(value::AGC, val::Dict{String, Any}) = value.ext = val
"""Set AGC internal."""
set_internal!(value::AGC, val::InfrastructureSystemsInternal) = value.internal = val


text:
#=
Adapted from Higgs-Boson ML competition on Kaggle

=#

function ams(s,b)
  #=
  Calculate the Approximate Median Significance defined as:

  AMS = sqrt( 2 * { (s + b + b_r) * log[1+(s/(b+b_r))] - s} )

  where b_r = 10, b = background, s = signal

  =#
  br = 10.0
  radicand = 2 * ((s+b+br) * log(1.0 + s/(b+br)) - s)
  if radicand < 0
    println("radicand is negative. Exiting.")
  else
    return sqrt(radicand)
  end
end

max_stars_repo_name: potterzot/jstats

max_issues_repo_name: potterzot/jstats

max_forks_repo_name: potterzot/jstats


text:
#
# Polarimetric_Parameters.jl
#
# Provide the type polarimetric parameter.
#
# ------------------------------------------------
#
# This file is part of Rhapsodie
#
#
# Copyright (c) 2017-2021 Laurence Denneulin (see LICENCE.md)

#------------------------------------------------
# Structure definition

    #For pixles
    struct PolarimetricPixel{T<: AbstractFloat}
        parameter_type::AbstractString   #either "stokes", "intensities" or "mixed"
        I::T
        Q::T
        U::T
        Iu::T
        Ip::T
        θ::T
    end

   #For map of pixels
    struct PolarimetricMap{T<: AbstractFloat}
        parameter_type::AbstractString   #either "stokes", "intensities" or "mixed"
        I::Array{T,2};
        Q::Array{T,2};
        U::Array{T,2};
        Iu::Array{T,2};
        Ip::Array{T,2};
        θ::Array{T,2};
    end

    PolarimetricMap(parameter_type::AbstractString,
                    I::AbstractArray{T,2},
                    Q::AbstractArray{T,2},
                    U::AbstractArray{T,2},
                    Iu::AbstractArray{T,2},
                    Ip::AbstractArray{T,2},
                    θ::AbstractArray{T,2}) where {T<:AbstractFloat} =       PolarimetricMap(parameter_type,
                                                         convert(Array{T},I),
                                                         convert(Array{T},Q),
                                                         convert(Array{T},U),
                                                         convert(Array{T},I),
                                                         convert(Array{T},Ip),
                                                         convert(Array{T},θ))

#------------------------------------------------
# Constructors
"""
    PolarimetricPixel(parameter_type, x) -> PolarimetricPixel
    PolarimetricMap(parameter_type, x) -> PolarimetricMap

create an object of type PolarimetricParameter from either:
    - Parameters I, Q, U (i.e. parameter_type = 'stokes')
    - Parameters Iu, Ip and θ (i.e. parameter_type = 'intensities')
    - Parameters Iu, Q, U (i.e. parameter_type = 'mixed')

Each parameter can be called from the structur. For exemple with a
construction from Stokes parameters S=(I,Q,U):

    using Rhapsodie
    X = PolarimetricParameter(S, 'stokes');
    X.I #yields the Stokes parameter I
    X.Ip #yields the polarized intensity Ip
    X[1,1] #yields a PolarimetricPix at the CartesianIndex (1,1);
    X[1,1].I #yields the Stokes parameter I at the CartesianIndex (1,1);

    PolarimetricMap(parameter_type, n1, n2) -> PolarimetricMap

yields an empty
"""
    function PolarimetricPixel(parameter_type::AbstractString,
                             x1::T,
                             x2::T,
                             x3::T) where {T<:AbstractFloat}
        if parameter_type == "stokes"
            I = x1            # Stokes parameter I (total light intensity)
            Q = x2            # Stokes parameter Q
            U = x3            # Stokes parameter U
            Ip = sqrt(Q^2 + U^2)    # intensity of linearly polarized light
            Iu = I-Ip    # intensity of unpolarized light
            θ = atan(U, Q)/2 # angle of linearly polarized light
        elseif parameter_type == "intensities"
            Iu = x1          # intensity of unpolarized light
            Ip = x2          # intensity of linearly polarized light
            θ = x3       # angle of linearly polarized light
            I = Iu + Ip   # Stokes parameter I (total light intensity)
            Q = Ip *cos(2*θ);    # Stokes parameter Q
            U = Ip *sin(2*θ);    # Stokes parameter U
        elseif parameter_type == "mixed"
            Iu = x1          # intensity of unpolarized light
            Q = x2            # Stokes parameter Q
            U = x3            # Stokes parameter U
            Ip = sqrt(Q^2 + U^2)    # intensity of linearly polarized light
            I = Iu + Ip     # Stokes parameter I (total light intensity
            θ = atan(U, Q)/2 # angle of linearly polarized light
        else
            error("unkown type, only known types : stokes, intensities and mixed")
        end
        PolarimetricPixel(parameter_type,I,Q,U,Iu,Ip,θ)
    end

    function PolarimetricPixel(parameter_type::AbstractString,
                             x::N) where {N<:AbstractArray{Float64,1}}
        @assert length(x) == 3
        PolarimetricPixel(parameter_type,
                        x[1],
                        x[2],
                        x[3]);
    end

    function PolarimetricMap(x::Array{PolarimetricPixel,2})
        T = Float64;
        n1, n2 = size(x)
        par_type=x[1].parameter_type;
        I = Array{T}(undef, n1, n2)    # Stokes parameter I (total light intensity)
        Q = Array{T}(undef, n1, n2)    # Stokes parameter Q
        U = Array{T}(undef, n1, n2)    # Stokes parameter U
        Iu = Array{T}(undef, n1, n2)    # intensity of unpolarized light
        Ip = Array{T}(undef, n1, n2)    # intensity of linearly polarized light
        θ = Array{T}(undef, n1, n2) # angle of linearly polarized light
        @inbounds for i2 in 1:n2
            @simd for i1 in 1:n1
                pix= x[i1,i2];
                if par_type == pix.parameter_type
                   I[i1,i2] = pix.I;
                   Q[i1,i2] = pix.Q;
                   U[i1,i2] = pix.U;
                   Iu[i1,i2] = pix.Iu;
                   Ip[i1,i2] = pix.Ip;
                   θ[i1,i2] = pix.θ;
                else
                   error("polarimetric types must match")
                end
            end
        end
        PolarimetricMap(par_type,I,Q,U,Iu,Ip,θ)
    end

    function PolarimetricMap(parameter_type::AbstractString,
                             x1::A,
                             x2::A,
                             x3::A) where {T<:AbstractFloat, A<:AbstractArray{T,2}}
        n1, n2 = size(x1)
        @assert ((n1,n2) == size(x2)) && ((n1,n2) == size(x3))
            if parameter_type == "stokes"
            I = x1            # Stokes parameter I (total light intensity)
            Q = x2            # Stokes parameter Q
            U = x3            # Stokes parameter U
            Iu = Array{T}(undef, n1, n2)    # intensity of unpolarized light
            Ip = Array{T}(undef, n1, n2)    # intensity of linearly polarized light
            θ = Array{T}(undef, n1, n2) # angle of linearly polarized light
            @inbounds for i2 in 1:n2
                @simd for i1 in 1:n1
                    Ip[i1,i2] = sqrt(Q[i1,i2]^2 + U[i1,i2]^2);
                    Iu[i1,i2] = I[i1, i2]-Ip[i1,i2];
                    θ[i1,i2] = atan(U[i1,i2], Q[i1,i2])/2;
                end
            end
        elseif parameter_type == "intensities"
            I = Array{T}(undef, n1, n2)    # Stokes parameter I (total light intensity)
            Q = Array{T}(undef, n1, n2)    # Stokes parameter Q
            U = Array{T}(undef, n1, n2)    # Stokes parameter U
            Iu = x1          # intensity of unpolarized light
            Ip = x2          # intensity of linearly polarized light
            θ = x3       # angle of linearly polarized light
             @inbounds for i2 in 1:n2
                @simd for i1 in 1:n1
                    I[i1,i2] = Iu[i1,i2] + Ip[i1,i2];
                    Q[i1,i2] = Ip[i1,i2] .*cos.(2*θ[i1,i2]);
                    U[i1,i2] = Ip[i1,i2] .*sin.(2*θ[i1,i2]);
                end
            end
        elseif parameter_type == "mixed"
            I = Array{T}(undef, n1, n2)     # Stokes parameter I (total light intensity
            Q = x2            # Stokes parameter Q
            U = x3            # Stokes parameter U
            Iu = x1          # intensity of unpolarized light
            Ip = Array{T}(undef, n1, n2)    # intensity of linearly polarized light
            θ = Array{T}(undef, n1, n2) # angle of linearly polarized light
            @inbounds for i2 in 1:n2
                @simd for i1 in 1:n1
                    Ip[i1,i2] = sqrt(Q[i1,i2]^2 + U[i1,i2]^2);
                    I[i1,i2] = Iu[i1,i2] + Ip[i1,i2];
                    θ[i1,i2] = atan(U[i1,i2], Q[i1,i2])/2;
                end
            end

        else
            error("unkown type, only known types : stokes, intensities and mixed")
        end
        PolarimetricMap(parameter_type,I,Q,U,Iu,Ip,θ)
    end

    function PolarimetricMap(parameter_type::AbstractString,
                             x::Array{T,3}) where {T<:AbstractFloat}
        n1, n2, n3 = size(x)
        @assert n3 == 3
        PolarimetricMap(parameter_type,
                        view(x,:,:,1),
                        view(x,:,:,2),
                        view(x,:,:,3));
    end

    function PolarimetricMap(parameter_type::AbstractString, n1::Int, n2::Int)
        return PolarimetricMap(parameter_type,
                               Array{Float64,2}(undef, n1, n2),
                               Array{Float64,2}(undef, n1, n2),
                               Array{Float64,2}(undef, n1, n2),
                               Array{Float64,2}(undef, n1, n2),
                               Array{Float64,2}(undef, n1, n2),
                               Array{Float64,2}(undef, n1, n2))
    end

#------------------------------------------------
# Base fonction redefinitions

    Base.size(A::PolarimetricMap) = size(A.I)
    Base.length(A::PolarimetricMap) =prod(size(A))*3
    Base.length(A::PolarimetricPixel) =3
    Base.getindex(X::PolarimetricMap, i::CartesianIndex{2}) where {N} =
    PolarimetricPixel(X.parameter_type, X.I[i], X.Q[i], X.U[i], X.Iu[i], X.Ip[i], X.θ[i])
    Base.getindex(X::PolarimetricMap, i::Int) =
    PolarimetricPixel(X.parameter_type, X.I[i], X.Q[i], X.U[i], X.Iu[i], X.Ip[i], X.θ[i])
    Base.getindex(X::PolarimetricMap, i::Int, j::Int) where {T<:Tuple} =
    getindex(X, CartesianIndex(i,j))

    function Base.setindex!(X::PolarimetricMap{Float64}, x::PolarimetricPixel{Float64}, i::Int64, j::Int64)
        X.I[i,j]=x.I;
        X.Q[i,j]=x.Q;
        X.U[i,j]=x.U;
        X.Iu[i,j]=x.Iu;
        X.Ip[i,j]=x.Ip;
        X.θ[i,j]=x.θ;
    end

    +(x::PolarimetricMap, y::PolarimetricMap) = PolarimetricMap(x.parameter_type,
                                                                x.I + y.I,
                                                                x.Q + y.Q,
                                                                x.U + y.U,
                                                                x.Iu + y.Iu,
                                                                x.Ip + y.Ip,
                                                                x.θ + y.θ)

     -(x::PolarimetricMap, y::PolarimetricMap) = PolarimetricMap(x.parameter_type,
                                                                x.I - y.I,
                                                                x.Q - y.Q,
                                                                x.U - y.U,
                                                                x.Iu - y.Iu,
                                                                x.Ip - y.Ip,
                                                                x.θ - y.θ)

     vcopy(x::PolarimetricMap) = PolarimetricMap(x.parameter_type,
                                                 x.I,
                                                 x.Q,
                                                 x.U,
                                                 x.Iu,
                                                 x.Ip,
                                                 x.θ)
     function vcreate(x::PolarimetricMap)

        @assert (x.parameter_type == "stokes") |
                (x.parameter_type == "intensities") |
                (x.parameter_type == "mixed")
         n1,n2=size(x);
         return PolarimetricMap(x.parameter_type, n1, n2)
     end

     function +(x::PolarimetricMap, y::Array{T,3}) where {T<:AbstractFloat}
        @assert size(y)[1:2] == size(x)
        if x.parameter_type == "stokes"
           I=x.I + view(y,:,:,1);
           Q=x.Q + view(y,:,:,2);
           U=x.U + view(y,:,:,3);
           return PolarimetricMap("stokes", I, Q, U)
        elseif x.parameter_type == "intensities"
           Iu=x.Iu + view(y,:,:,1);
           Ip=x.Ip + view(y,:,:,2);
           θ=x.θ + view(y,:,:,3);
           return PolarimetricMap("intensities", Iu, Ip, θ)
        elseif x.parameter_type == "mixed"
           Iu=x.Iu + view(y,:,:,1);
           Q=x.Q + view(y,:,:,2);
           U=x.U + view(y,:,:,3);
           return PolarimetricMap("mixed", Iu, Q, U)
        else
            error("unknown parameter type")
        end
     end

     +(y::Array{T,3}, x::PolarimetricMap) where {T<:AbstractFloat} = x + y
     -(x::PolarimetricMap, y::Array{T,3}) where {T<:AbstractFloat} = x + (-y)

     function convert(::Type{Array{T,3}}, x::PolarimetricMap{T}) where {T <:AbstractFloat}
         if x.parameter_type == "stokes"
           return cat(x.I, x.Q, x.U, dims=3)
        elseif x.parameter_type == "intensities"
           return cat(x.Iu, x.Ip, x.θ, dims=3)
        elseif x.parameter_type == "mixed"
           return cat(x.Iu, x.Q, x.U, dims=3)
        else
            error("unknown parameter type")
        end
     end

#------------------------------------------------
# Writting function to save PolarimetricMap in fits file
"""
    write(X,'filename.fits')

where X is a PolarimetricMap, write a fitsfile

"""

function write(X::PolarimetricMap, filename::AbstractString)
    data=cat(X.Iu', X.Ip', X.θ', X.I', X.Q', X.U',dims=3)
    header=FitsHeader()#;MAPORDER = (2, "Iu, Ip, theta, I, Q, U"))
    fitsdata=FitsImage(data)#, hdr=header)

    write!(filename, fitsdata)
end

"""
    read('parameter_type','filename.fits') -> PolarimetricMap

create an object of type PolarimetricMap from a fits file with:
    - Parameters I, Q, U (i.e. parameter_type = 'stokes')
    - Parameters Iu, Ip and θ (i.e. parameter_type = 'intensities')
    - Parameters Iu, Q, U (i.e. parameter_type = 'mixed')

"""

function read(parameter_type::AbstractString, filename::AbstractString)
    X=read(FitsArray, filename);
    return PolarimetricMap(parameter_type,
                           view(X,:,:,4)',
                           view(X,:,:,5)',
                           view(X,:,:,6)',
                           view(X,:,:,1)',
                           view(X,:,:,2)',
                           view(X,:,:,3)')
end

max_stars_repo_name: LaurenceDenneulin/Rhapsodie

max_issues_repo_name: LaurenceDenneulin/Rhapsodie

max_forks_repo_name: LaurenceDenneulin/Rhapsodie


text:
using JpegTurbo
using Test, TestImages

import JpegTurbo: LibJpeg

@testset "decompress" begin
    filename = testimage("earth_apollo17", download_only=true)

    cinfo = LibJpeg.jpeg_decompress_struct()
    jerr = Ref{LibJpeg.jpeg_error_mgr}()
    cinfo.err = LibJpeg.jpeg_std_error(jerr)
    LibJpeg.jpeg_create_decompress(cinfo)
    infile = ccall(:fopen, Libc.FILE, (Cstring, Cstring), filename, "rb")
    LibJpeg.jpeg_stdio_src(cinfo, infile)
    LibJpeg.jpeg_read_header(cinfo, true)
    LibJpeg.jpeg_start_decompress(cinfo)

    @test cinfo.output_width == 3000
    @test cinfo.output_height== 3002

    LibJpeg.jpeg_destroy_decompress(cinfo)
    ccall(:fclose, Cint, (Ptr{Libc.FILE},), infile)
end


text:
@testset "Print" begin
    buf = IOBuffer()

    print(buf, dna"")
    @test String(take!(buf)) == ""

    print(buf, dna"ACGTN")
    @test String(take!(buf)) == "ACGTN"

    print(buf, rna"ACGUN")
    @test String(take!(buf)) == "ACGUN"

    print(buf, dna"A"^100)
    @test String(take!(buf)) == "A"^100

    print(buf, dna"G"^60, width=0)
    @test String(take!(buf)) == "G"^60

    print(buf, dna"A"^60, width=-5)
    @test String(take!(buf)) == "A"^60

    print(buf, dna"A"^100, width=70)
    @test String(take!(buf)) == string("A"^70, '\n', "A"^30)

    print(buf, dna"A"^100, width=50)
    @test String(take!(buf)) == string("A"^50, '\n', "A"^50)

    print(buf, dna"A"^4100, width=100)
    @test String(take!(buf)) == repeat("A"^100 * '\n', 40) * "A"^100

    print(buf, char"ϐ")
    @test String(take!(buf)) == "ϐ"

    print(buf, char"ϐaδ", width=1)
    @test String(take!(buf)) == "ϐ\na\nδ"
end

@testset "Show" begin
    buf = IOBuffer()

    show(buf, dna"")
    @test String(take!(buf)) == "< EMPTY SEQUENCE >"

    show(buf, dna"ATCG")
    @test String(take!(buf)) == "ATCG"
end


text:
# code snipper to download git repository entirely (instead of zip file from site)

using GitCommand
git() do git
    downloadfolder = joinpath(homedir(),"Desktop")
    cd(downloadfolder)
    run(`$git clone https://github.com/B4rtDC/DS425-PS.git`)
end


text:
#=
Copyright (c) 2015, Intel Corporation
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
- Redistributions of source code must retain the above copyright notice,
  this list of conditions and the following disclaimer.
- Redistributions in binary form must reproduce the above copyright notice,
  this list of conditions and the following disclaimer in the documentation
  and/or other materials provided with the distribution.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
THE POSSIBILITY OF SUCH DAMAGE.
=#

"""
A collection of helper functions for processing AST nodes
"""
module Helper

import Base.hash
import Base.isequal

export LHSVar, RHSVar, TypedVar, LHSRealVar, toTypedVar
export TypedExpr, isArrayType, isInvoke, isCall, isTopNode, toLHSVar, toLHSVarOrNum, toLHSVarOrInt, isbitstuple, isPtrType, isIntType
export isBitArrayType, isTupleType, isStringType, isequal, hasSymbol, hash, isfunctionhead, isBaseFunc
export getCallFunction, getCallArguments, isAssignmentNode

if VERSION > v"0.5.0-dev+3260"
  if VERSION >= v"0.5.0-dev+3875"
    typealias GenSym     SSAValue
    export GenSym
    if VERSION >= v"0.5.0-dev+4094"
      immutable TopNode
        name :: Symbol
      end
      export TopNode
    end
  end
typealias LHSRealVar SlotNumber
typealias LHSVar     Union{SlotNumber, GenSym}
typealias RHSVar     Union{SlotNumber, TypedSlot, GenSym}
typealias TypedVar   TypedSlot
toLHSVar(tv::TypedVar) = SlotNumber(tv.id)
toLHSVar(tv::SlotNumber) = tv
isequal(x :: TypedVar, y :: TypedVar) = isequal(x.id, y.id) && isequal(x.typ, y.typ)
hash(x :: TypedVar) = hash(x.id)
toTypedVar(id::LHSRealVar, typ::DataType) = TypedSlot(id.id, typ)
else
typealias LHSRealVar Symbol
typealias LHSVar     Union{Symbol, GenSym}
typealias RHSVar     Union{Symbol, SymbolNode, GenSym}
typealias TypedVar   SymbolNode
typealias LambdaInfo LambdaStaticData
toLHSVar(x :: Symbol) = x
toLHSVar(tv::TypedVar) = tv.name
isequal(x :: TypedVar, y :: TypedVar) = isequal(x.name, y.name) && isequal(x.typ, y.typ)
hash(x :: TypedVar) = hash(x.name)
toTypedVar(id::LHSRealVar, typ::DataType) = SymbolNode(id, typ)
export LambdaInfo
end

"""
This should always be used instead of Expr(...) to form an expression as it forces the typ to be provided.
"""
function TypedExpr(typ, rest...)
    res = Expr(rest...)
    res.typ = typ
    res
end

"""
Returns true if the incoming type in "typ" is an array type.
"""
isArrayType(typ::DataType) = (typ<:Array) || (typ<:BitArray)
isArrayType(x::ANY) = false
isBitArrayType(typ::DataType) = typ<:BitArray
isBitArrayType(x::ANY) = false
isPtrType(typ::DataType) = typ<:Ptr
isPtrType(typ::ANY) = false
isInvoke(node::Expr) = node.head==:invoke
isInvoke(node::Any) = false
isCall(node::Expr) = node.head==:call
isCall(node::Any) = false
isTopNode(node::TopNode) = true
isTopNode(node::Any) = false

"""
TopNode is removed in recent 0.5 versions, we need a backward compatible way to compare
them, or their equivalent Base functions.
"""
isBaseFunc(node::TopNode, name::Symbol) = node.name == name
function isBaseFunc(node::GlobalRef, name::Symbol)
  m = node.mod
  if node.name == name
    while m != Base && m != Core && m != Main
      m = module_parent(m)
    end
    return m == Base || m == Core
  else
    return false
  end
end

isBaseFunc(node, name) = false

"""
In various places we need a LHSVar type which is the union of Symbol and GenSym.
This function takes a Symbol, SymbolNode, or GenSym and return either a Symbol or GenSym.
"""
toLHSVar(x :: GenSym) = x
toLHSVar(x :: Int) = x
toLHSVar(x) = x
toLHSVarOrNum(x :: RHSVar) = toLHSVar(x)
toLHSVarOrNum(x :: Number) = x
toLHSVarOrInt(x :: RHSVar) = toLHSVar(x)
toLHSVarOrInt(x :: Int) = x

"""
Returns true if input "a" is a tuple and each element of the tuple of isbits type.
"""
function isbitstuple(a::Tuple)
    for i in a
        if !isbits(i)
            return false
        end
    end
    return true
end

isbitstuple(a::Any) = false
isIntType(typ::DataType) = typ<:Integer
isIntType(typ::ANY) = false
isTupleType(typ::DataType) = typ <: Tuple
isTupleType(typ::ANY) = false
isStringType(typ::DataType) = typ <: AbstractString
isStringType(typ::ANY) = false
hasSymbol(ssn :: Symbol) = true
hasSymbol(ssn :: TypedVar) = true
hasSymbol(ssn :: Expr) = ssn.head == :(::)
hasSymbol(ssn) = false

if VERSION > v"0.5.0-dev+3260"
isfunctionhead(x) = isa(x, LambdaInfo)
else
isfunctionhead(x) = isa(x, Expr) && x.head == :lambda && isa(x.args[3], Expr) && x.args[3].head == :body
end

function print_by_field(obj)
  fields = fieldnames(obj)
  for f in fields
    try
      println(f, " = ", getfield(obj, f), " type = " , typeof(getfield(obj,f)))
    catch
      println("Failed for field ", f)
    end
  end
end

function getCallFunction(exp::Expr)
  if exp.head == :invoke
    exp.args[2]
  elseif exp.head == :call
    exp.args[1]
  else
    error(string("Expect a :call or :invoke Expr, but got ", exp))
  end
end

function getCallArguments(exp::Expr)
  if exp.head == :invoke
    exp.args[3:end]
  elseif exp.head == :call
    exp.args[2:end]
  else
    error(string("Expect a :call or :invoke Expr, but got ", exp))
  end
end

"""
Is a node an assignment expression node.
"""
isAssignmentNode(node::Expr) = node.head == :(=)
isAssignmentNode(node::Any)  = false

end # module Helper


text:
f = x -> begin
	N = lastindex(x)
    return 100sum((x[i+1] - x[i]^2)^2 for i = 1:N-1) + sum((x[i] - 1.0)^2 for i = 1:N-1)
end

g! = (g, x) -> begin
	N = lastindex(x)
	g[1] = -2*(1 - x[1]) - 400*x[1]*(-x[1]^2 + x[2])
	for i in 2:N-1
		g[i] = -2*(1 - x[i]) + 200*(-x[i-1]^2 + x[i]) - 400*x[i]*(-x[i]^2 + x[1 + i])
	end
	g[N] = 200 * (x[N] - x[N-1]^2)
	return g
end

fg! = (g, x) -> begin
	@warn "rosen fg! not implmented"
end

init = (n::Int=500) -> begin
	x0 = [j/(n+1) for j in 1:n]
	return n, x0
end

TestSet["rosen"] = UncProgram("rosen",  f, g!, fg!, init)


text:
using RecipesBase

export SnfPosteriorSampler, SnfPosteriorMcmcOutput
export get_sample_matrices

struct SnfAuxiliaryMcmc{T,N,S<:McmcSampler}
    mcmc::S  # Mcmc move (leave general since will also use for graph models)
    data::Vector{Array{T,N}} # To store auxiliary samples
    init_at_prev::Bool
end

# We need to specify (i) if vectorised (ii) element types.
# (Both will will change how the auxiliary data is sampled)
struct SnfPosteriorSampler{T<:Union{Int,Bool},N,S₁<:McmcMove,S₂<:McmcSampler}
    move::S₁    # Move for mode update
    ε::Float64 # Neighborhood for gamma proposal
    aux::SnfAuxiliaryMcmc{T,N,S₂}  # auxiliary MCMC sampler
    output::McmcOutputParameters
    suff_stats::Vector{Float64}
    γ_counts::Vector{Int}
    function SnfPosteriorSampler(
        move::Q, aux_mcmc::McmcSampler,
        posterior::SnfPosterior{T,N,V,S};
        ε::Float64=0.15,
        desired_samples::Int=1000, burn_in::Int=0, lag::Int=1,
        aux_init_at_prev::Bool=false,
        ) where {Q<:McmcMove,T,N,V,S}
        suff_stats = [0.0,0.0]
        aux_data = Array{T,N}[]
        γ_counts = [0,0]
        aux = SnfAuxiliaryMcmc(aux_mcmc, aux_data, aux_init_at_prev)
        output = McmcOutputParameters(desired_samples, burn_in, lag)
        new{T,N,Q,typeof(aux_mcmc)}(
            move, ε, aux,
            output,
            suff_stats,
            γ_counts
            )
    end
end

Base.show(io::IO, x::SnfPosteriorSampler) = print(io, typeof(x))

struct SnfPosteriorMcmcOutput{T,N,V,S}
    x_sample::Vector{Array{T,N}}
    γ_sample::Vector{Float64}
    suff_stat_trace::Vector{Float64}
    posterior::SnfPosterior{T,N,V,S}
end

Base.show(io::IO, x::SnfPosteriorMcmcOutput{T,N,V,S}) where {T,N,V,S} = print(io, typeof(x))

@recipe function f(
    output::SnfPosteriorMcmcOutput{T,N,V,S},
    x_true::Array{T,N}
    ) where {T,N,V,S}

    x_sample = output.x_sample
    γ_sample = output.γ_sample
    layout := (2,1)
    legend --> false
    xguide --> "Index"
    yguide --> ["Distance from True Mode" "γ"]
    size --> (800, 600)
    margin --> 5mm
    y1 = map(x->output.posterior.d(x_true,x), x_sample)
    y2 = γ_sample
    hcat(y1,y2)
end

function get_sample_matrices(
    output::SnfPosteriorMcmcOutput{T,N,V,S}
    ) where {T,N,V,S}
    if N == 1
        dir, sl = (
            output.posterior.directed,
            output.posterior.self_loops
        )
        return vec_to_adj_mat.(
            output.x_sample,
            directed=dir,
            self_loops=sl
        )
    elseif N==2
        return output.x_sample

    else
        error("Data of invalid dimension, should be 1d or 2d.")
    end
end

function eval_accept_prob(
    x_curr::Array{T,N},
    x_prop::Array{T,N},
    γ_curr::Float64,
    aux::SnfAuxiliaryMcmc{T,N,S},
    posterior::SnfPosterior{T,N,V,R},
    log_ratio::Float64,
    suff_stats::Vector{Float64}
    ) where {T,N,S,V,R}

    G_prior = posterior.G_prior
    d = posterior.d
    x_mode_prior, γ_prior = (
        G_prior.mode, G_prior.γ
    )

    # Infer aux model from mode prior
    aux_model = similar(
        G_prior,
        x_prop,
        γ_curr
    )

    if aux.init_at_prev
        @inbounds tmp = deepcopy(aux.data[end])
        draw_sample!(aux.data, aux.mcmc, aux_model, init=tmp)
    else
        draw_sample!(aux.data, aux.mcmc, aux_model)
    end

    aux_log_lik_ratio = -γ_curr * (
        sum(d(x, x_curr) for x in aux.data)
        - sum(d(x, x_prop) for x in aux.data)
    )

    suff_stats[2] = sum(d(x, x_prop) for x in posterior.data)

    log_lik_ratio = -γ_curr * (
        suff_stats[2] - suff_stats[1] # (prop-curr)
    )

    log_prior_ratio = -γ_prior * (
        d(x_prop, x_mode_prior) - d(x_curr, x_mode_prior)
    )

    log_α = log_lik_ratio + log_prior_ratio + aux_log_lik_ratio + log_ratio

    return log_α

end

function accept_reject_mode!(
    x_curr::Array{T,N},
    x_prop::Array{T,N},
    γ_curr::Float64,
    move::McmcMove,
    aux::SnfAuxiliaryMcmc{T,N,S},
    posterior::SnfPosterior{T,N,V,R},
    suff_stats::Vector{Float64}
    ) where {T,N,S,V,R}

    # Do imcmc proposal
    log_ratio = prop_sample!(x_curr, x_prop, move)

    log_α = eval_accept_prob(
        x_curr, x_prop, γ_curr,
        aux,
        posterior,
        log_ratio,
        suff_stats
    )

    move.counts[2] += 1

    if log(rand()) < log_α
        # We accept!
        move.counts[1] += 1
        suff_stats[1] = suff_stats[2]  # Update sufficient stat
        copy!(x_curr, x_prop)
    else
        # We reject!
        copy!(x_prop, x_curr)
    end
end

# Must specialise for Gibbs scan move (for vector)
function accept_reject_mode!(
    x_curr::Array{T,1},
    x_prop::Array{T,1},
    γ_curr::Float64,
    move::GibbsScanMove,
    aux::SnfAuxiliaryMcmc{T,1,S},
    posterior::SnfPosterior{T,1,V,R},
    suff_stats::Vector{Float64}
    ) where {T,S,V,R}

    move.counts[2] += length(x_curr)
    for i in eachindex(x_curr)
        log_ratio = prop_sample_entry!(x_curr, x_prop, move, i)
        log_α = eval_accept_prob(
            x_curr, x_prop, γ_curr,
            aux,
            posterior,
            log_ratio,
            suff_stats
        )

        if log(rand()) < log_α
            # We accept!
            move.counts[1] += 1
            suff_stats[1] = suff_stats[2]  # Update sufficient stat
            copy!(x_curr, x_prop)
        else
            # We reject!
            copy!(x_prop, x_curr)
        end
    end
end

# Wrapper to unpack the mcmc bits and pass...
function accept_reject_mode!(
    x_curr::Array{T,N},
    x_prop::Array{T,N},
    γ_curr::Float64,
    mcmc::SnfPosteriorSampler{T,N,S₁,S₂},
    posterior::SnfPosterior{T,N,V,R}
    ) where {T,N,S₁,S₂,V,R}

    accept_reject_mode!(
        x_curr, x_prop, γ_curr,
        mcmc.move,
        mcmc.aux,
        posterior,
        mcmc.suff_stats
    )
end

function initialise!(
    aux::SnfAuxiliaryMcmc{T,N,S},
    x_init::Array{T,N},
    γ_init::Float64,
    G_prior::SNF,
    n::Int # Sample size
    ) where {T,N,S<:McmcSampler}

    if length(aux.data) ≥ n
        resize!(aux.data, n)
    else
        append!(aux.data, [similar(x_init) for i in 1:(n-length(aux.data))])
    end
    aux_model = similar(
        G_prior,
        x_init,
        γ_init
    )
    draw_sample!(aux.data, aux.mcmc, aux_model)

end

function draw_sample_mode!(
    sample_out::Union{Vector{Vector{Int}}, SubArray},
    mcmc::SnfPosteriorSampler{Int,1,S₁,S₂},
    posterior::VecMultigraphSnfPosterior,
    γ_fixed::Float64;
    burn_in::Int=mcmc.burn_in,
    lag::Int=mcmc.lag,
    G_init::Union{Matrix{Int},Vector{Int}}=sample_frechet_mean(posterior.data_vec, posterior.d)[1],
    loading_bar::Bool=true
    ) where {S₁,S₂}

    if loading_bar
        iter = Progress(
            length(sample_out)*lag + burn_in, # How many iters
            1,  # At which granularity to update loading bar
            "Chain for γ = $(γ_fixed) and n = $(posterior.sample_size) (mode conditional)....")  # Loading bar. Minimum update interval: 1 second
    end

    # Initialise states
    if typeof(G_init)==Vector{Int}
        x_curr = copy(G_init)
        x_prop = copy(x_curr)
    elseif typeof(G_init)==Matrix{Int}
        directed, self_loops = (
            posterior.directed,
            posterior.self_loops
        )
        x_curr = adj_mat_to_vec(
            G_init,
            directed=directed,
            self_loops=self_loops
        )
        x_prop = copy(x_curr)
    else
        error("Initial value of unsupported type.")
    end

    sample_count = 1    # Keeps which sample to be stored we are working to get
    i = 0               # Keeps track all samples (included lags and burn_ins)
    reset_counts!(mcmc.move) # Reset counts for mcmc move (tracks acceptances)

    # Initialise aux mcmc (resize data and draw initial sample)
    initialise!(mcmc.aux, x_curr, γ_curr, posterior.G_prior, posterior.sample_size)

    # Initialise sufficient statistic
    d, data = (posterior.d, posterior.data)
    suff_stats = mcmc.suff_stats
    suff_stats[1] = sum(d(x_curr, x) for x in data)

    # Storage for sample statistic
    sample_suff_stat = zeros(length(sample_out))

    while sample_count ≤ length(sample_out)
        i += 1
        # Store value
        if (i > burn_in) & (((i-1) % lag)==0)
            @inbounds sample_out[sample_count] = deepcopy(x_curr)
            @inbounds sample_suff_stat[sample_count] = suff_stats[1]
            sample_count += 1
        end
        accept_reject_mode!(
            x_curr, x_prop,
            γ_fixed,
            mcmc, posterior
        )
        # println(S_prop)
        if loading_bar
            next!(iter)
        end
    end
    return sample_suff_stat
end

function draw_sample_mode(
    mcmc::SnfPosteriorSampler,
    posterior::VecMultigraphSnfPosterior,
    γ_fixed::Float64;
    desired_samples::Int=mcmc.desired_samples,
    args...
    )

    sample_out = [Int[] for i in 1:desired_samples]
    sample_suff_stat = draw_sample_mode!(
        sample_out,
        mcmc, posterior, γ_fixed,
        args...
    )

    return sample_out, sample_suff_stat

end

function (mcmc::SnfPosteriorSampler)(
    posterior::VecMultigraphSnfPosterior,
    γ_fixed::Float64;
    args...
    )

    x_sample, suff_stat_trace = draw_sample_mode(
        mcmc, posterior, γ_fixed,
        arg...
    )
    γ_sample = fill(γ_fixed, length(S_sample))

    # .... define output struct
    return SnfPosteriorMcmcOutput(
        x_sample,
        γ_sample,
        suff_stat_trace,
        posterior
    )
end

function accept_reject_gamma(
    γ_curr::Float64,
    x_curr::Vector{Int},
    mcmc::SnfPosteriorSampler,
    posterior::VecMultigraphSnfPosterior
    )

    G_prior = posterior.G_prior
    d = posterior.d
    aux = mcmc.aux
    ε, suff_stats = (
        mcmc.ε,
        mcmc.suff_stats
    )
    # Draw proposal
    γ_prop = rand_reflect(γ_curr, ε, 0.0, Inf)

    aux_model = similar(
        G_prior,
        x_curr,
        γ_prop
    )

    if aux.init_at_prev
        @inbounds tmp = deepcopy(aux.data[end])
        draw_sample!(aux.data, aux.mcmc, aux_model, init=tmp)
    else
        draw_sample!(aux.data, aux.mcmc, aux_model)
    end

    # Accept reject
    log_lik_ratio = (γ_curr - γ_prop) * suff_stats[1]
    aux_log_lik_ratio = (γ_prop - γ_curr) * sum(d(x,x_curr) for x in aux.data)

    log_α = (
        logpdf(posterior.γ_prior, γ_prop)
        - logpdf(posterior.γ_prior, γ_curr)
        + log_lik_ratio + aux_log_lik_ratio
    )
    if log(rand()) < log_α
        return γ_prop, 1
    else
        return γ_curr, 0
    end
end

function draw_sample_gamma!(
    sample_out::Union{Vector{Float64}, SubArray},
    mcmc::SnfPosteriorSampler,
    posterior::SnfPosterior,
    x_fixed::Vector{Int};
    burn_in::Int=mcmc.output.burn_in,
    lag::Int=mcmc.output.lag,
    γ_init::Float64=4.0,
    loading_bar::Bool=true
    )

    if loading_bar
        iter = Progress(
            length(sample_out)*lag + burn_in, # How many iters
            1,  # At which granularity to update loading bar
            "Chain for γ = $(γ_fixed) and n = $(posterior.sample_size) (mode conditional)....")  # Loading bar. Minimum update interval: 1 second
    end

    γ_curr = γ_init
    x_curr = copy(x_fixed)

    sample_count = 1    # Keeps which sample to be stored we are working to get
    i = 0               # Keeps track all samples (included lags and burn_ins)

    # Initialise aux mcmc (resize data and draw initial sample)
    initialise!(mcmc.aux, x_curr, γ_curr, posterior.G_prior, posterior.sample_size)

    acc_count = 0
    i = 0 # Counter for iterations
    sample_count = 1  # Which sample we are working to get

    while sample_count ≤ length(sample_out)
        i += 1
        # Store value
        if (i > burn_in) & (((i-1) % lag)==0)
            sample_out[sample_count] = γ_curr
            sample_count += 1
        end

        γ_curr, was_acc = accept_reject_gamma(
            γ_curr, x_curr,
            mcmc, posterior
        )
        acc_count += was_acc

        if loading_bar
            next!(iter)
        end

    end
    mcmc.γ_counts = [acc_count, i]
end

function draw_sample_gamma(
    mcmc::SnfPosteriorSampler,
    posterior::SnfPosterior,
    x_fixed::Vector{Int};
    desired_samples::Int=mcmc.output.desired_samples,
    args...
    )

    sample_out = zeros(desired_samples)
    draw_sample_gamma!(
        sample_out,
        mcmc, posterior,
        x_fixed,
        args...
        )
    return sample_out

end

function (mcmc::SnfPosteriorSampler)(
    posterior::SnfPosterior,
    x_fixed::Vector{Int};
    args...
    )

    γ_sample = draw_sample_gamma(
        mcmc, posterior, γ_fixed,
        args...
    )
    x_sample = [copy(x_fixed) for i in eachindex(γ_sample)]
    suff_stat = sum(posterior.di(x,x_fixed) for x in posterior.data)
    suff_stat_trace = fill(suff_stat, length(γ_sample))
    return SnfPosteriorMcmcOutput(
        x_sample, γ_sample,
        suff_stat_trace,
        posterior
    )
end

function draw_sample!(
    sample_out_x::T,
    sample_out_gamma::S,
    mcmc::SnfPosteriorSampler,
    posterior::SnfPosterior;
    burn_in::Int=mcmc.output.burn_in,
    lag::Int=mcmc.output.lag,
    G_init::Union{Matrix{Int},Vector{Int}}=sample_frechet_mean(posterior.data, posterior.d)[1],
    γ_init::Float64=5.0,
    loading_bar::Bool=true
    ) where {T<:Union{Vector{Vector{Int}},SubArray},S<:Union{Vector{Float64},SubArray}}

    if loading_bar
        iter = Progress(
            length(sample_out_x)*lag + burn_in, # How many iters
            1,  # At which granularity to update loading bar
            "Chain for n = $(posterior.sample_size) (joint)....")  # Loading bar. Minimum update interval: 1 second
    end

    # Initialise states
    if typeof(G_init)==Vector{Int}
        x_curr = copy(G_init)
        x_prop = copy(x_curr)
    elseif typeof(G_init)==Matrix{Int}
        directed, self_loops = (
            posterior.directed,
            posterior.self_loops
        )
        x_curr = adj_mat_to_vec(
            G_init,
            directed=directed,
            self_loops=self_loops
        )
        x_prop = copy(x_curr)
    else
        error("Initial value of unsupported type.")
    end
    # @show x_curr
    γ_curr = γ_init

    sample_count = 1    # Keeps which sample to be stored we are working to get
    i = 0               # Keeps track all samples (included lags and burn_ins)
    γ_acc_count = 0
    reset_counts!(mcmc.move) # Reset counts for mcmc move (tracks acceptances)

    # Initialise aux mcmc (resize data and draw initial sample)

    initialise!(mcmc.aux, x_curr, γ_curr, posterior.G_prior, posterior.sample_size)

    # Initialise sufficient statistic
    d, data = (posterior.d, posterior.data)
    suff_stats = mcmc.suff_stats
    suff_stats[1] = sum(d(x_curr, x) for x in data)

    # Storage for sample statistic
    sample_suff_stat = zeros(length(sample_out_x))

    while sample_count ≤ length(sample_out_x)
        i += 1
        # Store value
        if (i > burn_in) & (((i-1) % lag)==0)
            @inbounds sample_out_x[sample_count] = copy(x_curr)
            @inbounds sample_out_gamma[sample_count] = γ_curr
            @inbounds sample_suff_stat[sample_count] = suff_stats[1]
            sample_count += 1
        end
        # Accept-reject move

        accept_reject_mode!(
            x_curr, x_prop,
            γ_curr,
            mcmc, posterior
        )

        # Accept-reject dispersion
        γ_curr, was_acc = accept_reject_gamma(
            γ_curr, x_curr,
            mcmc,
            posterior
        )

        γ_acc_count += was_acc
        # println(S_prop)
        if loading_bar
            next!(iter)
        end
    end
    mcmc.γ_counts[1:2] = [γ_acc_count, i]
    return sample_suff_stat
end

function draw_sample(
    mcmc::SnfPosteriorSampler,
    posterior::SnfPosterior;
    desired_samples::Int=mcmc.output.desired_samples,
    args...
    )

    sample_out_x = Vector{Vector{Int}}(undef, desired_samples)
    sample_out_gamma = zeros(desired_samples)
    suff_stat_sample = draw_sample!(
        sample_out_x,
        sample_out_gamma,
        mcmc,
        posterior;
        args...
        )
    return sample_out_x, sample_out_gamma, suff_stat_sample
end

function (mcmc::SnfPosteriorSampler)(
    posterior::SnfPosterior;
    args...
    )
    x_sample, γ_sample, suff_stat_trace = draw_sample(
        mcmc, posterior;
        args...
    )
    return SnfPosteriorMcmcOutput(
        x_sample, γ_sample,
        suff_stat_trace,
        posterior
    )
end


text:
"""
    shaded_line(x, y, lower, upper)

Plots a line through `x` and `y` and a filled area
behind it defined by `lower` and `upper`
"""
@recipe(ShadedLine, x, y, lower, upper) do scene
    Theme(;
        default_theme(scene, Band)...,
        color = theme(scene, :color),
        fillalpha = 0.2
    )
end

function plot!(p::ShadedLine)
    x, y, lower, upper = p[1:4]
    theme = copy(Theme(p))
    theme[:color] = lift(combine_color_alpha, Theme(p)[:color], Theme(p)[:fillalpha])
    plot!(p, Band, theme, x, lower, upper)
    plot!(p, Lines, Theme(p), x, y)
end

max_stars_repo_name: piever/StatMakie

max_issues_repo_name: piever/StatMakie

max_forks_repo_name: piever/StatMakie


text:
# Support Python iterators in Julia and vice versa

#########################################################################
# Iterating over Python objects in Julia

Base.IteratorSize(::Type{PyObject}) = Base.SizeUnknown()
function _start(po::PyObject)
    disable_sigint() do
        o = PyObject(@pycheckn ccall((@pysym :PyObject_GetIter), PyPtr, (PyPtr,), po))
        nxt = PyObject(@pycheck ccall((@pysym :PyIter_Next), PyPtr, (PyPtr,), o))

        return (nxt,o)
    end
end

"""
    PyIterator{T}(pyobject)

Wrap `pyobject::PyObject` into an iterator, that produces items of type `T`. To be more precise `convert(T, item)` is applied in each iteration. This can be useful to avoid automatic conversion of items into corresponding julia types.
```jldoctest
julia> using PyCall

julia> l = PyObject([PyObject(1), PyObject(2)])
PyObject [1, 2]

julia> piter = PyCall.PyIterator{PyAny}(l)
PyCall.PyIterator{PyAny,Base.HasLength()}(PyObject [1, 2])

julia> collect(piter)
2-element Array{Any,1}:
    1
    2

julia> piter = PyCall.PyIterator(l)
PyCall.PyIterator{PyObject,Base.HasLength()}(PyObject [1, 2])

julia> collect(piter)
2-element Array{PyObject,1}:
    PyObject 1
    PyObject 2
```
"""
struct PyIterator{T,S}
    o::PyObject
end

function _compute_IteratorSize(o::PyObject)
    S = try
        length(o)
        Base.HasLength
    catch err
        if !(err isa PyError && pyisinstance(err.val, @pyglobalobjptr :PyExc_TypeError))
            rethrow()
        end
        Base.SizeUnknown
    end
end
function PyIterator(o::PyObject)
    PyIterator{PyObject}(o)
end
function (::Type{PyIterator{T}})(o::PyObject) where {T}
    S = _compute_IteratorSize(o)
    PyIterator{T,S}(o)
end

Base.eltype(::Type{<:PyIterator{T}}) where T = T
Base.eltype(::Type{<:PyIterator{PyAny}}) = Any
Base.length(piter::PyIterator) = length(piter.o)

Base.IteratorSize(::Type{<: PyIterator{T,S}}) where {T,S} = S()

_start(piter::PyIterator) = _start(piter.o)

function Base.iterate(piter::PyIterator{T}, s=_start(piter)) where {T}
    ispynull(s[1]) && return nothing
    disable_sigint() do
        nxt = PyObject(@pycheck ccall((@pysym :PyIter_Next), PyPtr, (PyPtr,), s[2]))
        return (convert(T,s[1]), (nxt, s[2]))
    end
end
function Base.iterate(po::PyObject, s=_start(po))
    # avoid the constructor that calls length
    # since that might be an expensive operation
    # even if length is cheap, this adds 10% performance
    piter = PyIterator{PyAny, Base.SizeUnknown}(po)
    iterate(piter, s)
end

# issue #216
function Base.collect(::Type{T}, o::PyObject) where T
    a = T[]
    for x in o
        push!(a, x)
    end
    return a
end
Base.collect(o::PyObject) = collect(Any, o)

#########################################################################
# Broadcasting: if the object is iterable, return collect(o), and otherwise
#               return o.
function Base.Broadcast.broadcastable(o::PyObject)
    iter = ccall((@pysym :PyObject_GetIter), PyPtr, (PyPtr,), o)
    if iter == C_NULL
        pyerr_clear()
        return Ref(o)
    else
        ccall(@pysym(:Py_DecRef), Cvoid, (PyPtr,), iter)
        return collect(o)
    end
end


text:
# ---
# title: 817. Linked List Components
# id: problem817
# author: Tian Jun
# date: 2020-10-31
# difficulty: Medium
# categories: Linked List
# link: <https://leetcode.com/problems/linked-list-components/description/>
# hidden: true
# ---
#
# We are given `head`, the head node of a linked list containing  **unique
# integer values**.
#
# We are also given the list `G`, a subset of the values in the linked list.
#
# Return the number of connected components in `G`, where two values are
# connected if they appear consecutively in the linked list.
#
# **Example 1:**
#
#
#
#     Input:
#     head: 0->1->2->3
#     G = [0, 1, 3]
#     Output: 2
#     Explanation:
#     0 and 1 are connected, so [0, 1] and [3] are the two connected components.
#
#
# **Example 2:**
#
#
#
#     Input:
#     head: 0->1->2->3->4
#     G = [0, 3, 1, 4]
#     Output: 2
#     Explanation:
#     0 and 1 are connected, 3 and 4 are connected, so [0, 1] and [3, 4] are the two connected components.
#
#
# **Note:**
#
#   * If `N` is the length of the linked list given by `head`, `1 <= N <= 10000`.
#   * The value of each node in the linked list will be in the range` [0, N - 1]`.
#   * `1 <= G.length <= 10000`.
#   * `G` is a subset of all values in the linked list.
#
#
## @lc code=start
using LeetCode

## add your code here:
## @lc code=end