text:
const OBJ_METADATA = IdDict{Any, Dict{String, String}}()

function setmetadata!(x, meta::Dict{String, String})
    OBJ_METADATA[x] = meta
    return
end

getmetadata(x, default=nothing) = get(OBJ_METADATA, x, default)

"""
    Arrow.write(io::IO, tbl)
    Arrow.write(file::String, tbl)

Write any Tables.jl-compatible `tbl` out as arrow formatted data.
Providing an `io::IO` argument will cause the data to be written to it
in the "streaming" format, unless `file=true` keyword argument is passed.
Providing a `file::String` argument will result in the "file" format being written.

Multiple record batches will be written based on the number of
`Tables.partitions(tbl)` that are provided; by default, this is just
one for a given table, but some table sources support automatic
partitioning. Note you can turn multiple table objects into partitions
by doing `Tables.partitioner([tbl1, tbl2, ...])`, but note that
each table must have the exact same `Tables.Schema`.

By default, `Arrow.write` will use multiple threads to write multiple
record batches simultaneously (e.g. if julia is started with `julia -t 8`).

Supported keyword arguments to `Arrow.write` include:
  * `compress::Symbol`: possible values include `:lz4` or `:zstd`; will cause all buffers in each record batch to use the respective compression encoding
  * `dictencode::Bool=false`: whether all columns should use dictionary encoding when being written
  * `dictencodenested::Bool=false`: whether nested data type columns should also dict encode nested arrays/buffers; many other implementations don't support this
  * `denseunions::Bool=true`: whether Julia `Vector{<:Union}` arrays should be written using the dense union layout; passing `false` will result in the sparse union layout
  * `largelists::Bool=false`: causes list column types to be written with Int64 offset arrays; mainly for testing purposes; by default, Int64 offsets will be used only if needed
  * `file::Bool=false`: if a an `io` argument is being written to, passing `file=true` will cause the arrow file format to be written instead of just IPC streaming
"""
function write end

function write(file::String, tbl; largelists::Bool=false, compress::Union{Nothing, Symbol}=nothing, denseunions::Bool=true, dictencode::Bool=false, dictencodenested::Bool=false)
    open(file, "w") do io
        write(io, tbl, true, largelists, compress, denseunions, dictencode, dictencodenested)
    end
    return file
end

function write(io::IO, tbl; largelists::Bool=false, compress::Union{Nothing, Symbol}=nothing, denseunions::Bool=true, dictencode::Bool=false, dictencodenested::Bool=false, file::Bool=false)
    return write(io, tbl, file, largelists, compress, denseunions, dictencode, dictencodenested)
end

@static if VERSION >= v"1.3"
    const Cond = Threads.Condition
else
    const Cond = Condition
end

struct OrderedChannel{T}
    chan::Channel{T}
    cond::Cond
    i::Ref{Int}
end

OrderedChannel{T}(sz) where {T} = OrderedChannel{T}(Channel{T}(sz), Threads.Condition(), Ref(1))
Base.iterate(ch::OrderedChannel, st...) = iterate(ch.chan, st...)

macro lock(obj, expr)
    esc(quote
    @static if VERSION >= v"1.3"
        lock($obj)
    end
        try
            $expr
        finally
            @static if VERSION >= v"1.3"
                unlock($obj)
            end
        end
    end)
end

function Base.put!(ch::OrderedChannel{T}, x::T, i::Integer, incr::Bool=false) where {T}
    @lock ch.cond begin
        while ch.i[] < i
            wait(ch.cond)
        end
        put!(ch.chan, x)
        if incr
            ch.i[] += 1
        end
        notify(ch.cond)
    end
    return
end

function Base.close(ch::OrderedChannel)
    @lock ch.cond begin
        while Base.n_waiters(ch.cond) > 0
            wait(ch.cond)
        end
        close(ch.chan)
    end
    return
end

function write(io, source, writetofile, largelists, compress, denseunions, dictencode, dictencodenested)
    if writetofile
        @debug 1 "starting write of arrow formatted file"
        Base.write(io, "ARROW1\0\0")
    end
    msgs = OrderedChannel{Message}(Inf)
    # build messages
    sch = Ref{Tables.Schema}()
    firstcols = Ref{Any}()
    dictencodingvalues = Dict{Int, Set}()
    blocks = (Block[], Block[])
    # start message writing from channel
@static if VERSION >= v"1.3-DEV"
    tsk = Threads.@spawn for msg in msgs
        Base.write(io, msg, blocks, sch)
    end
else
    tsk = @async for msg in msgs
        Base.write(io, msg, blocks, sch)
    end
end
    @sync for (i, tbl) in enumerate(Tables.partitions(source))
        @debug 1 "processing table partition i = $i"
        if i == 1
            cols = toarrowtable(tbl, largelists, compress, denseunions, dictencode, dictencodenested)
            sch[] = Tables.schema(cols)
            firstcols[] = cols
            put!(msgs, makeschemamsg(sch[], cols), i)
            if !isempty(cols.dictencodings)
                for de in cols.dictencodings
                    dictencodingvalues[de.id] = Set(z for z in de.data)
                    dictsch = Tables.Schema((:col,), (eltype(de.data),))
                    put!(msgs, makedictionarybatchmsg(dictsch, (col=de.data,), de.id, false), i)
                end
            end
            put!(msgs, makerecordbatchmsg(sch[], cols), i, true)
        else
@static if VERSION >= v"1.3-DEV"
            Threads.@spawn begin
                try
                    cols = toarrowtable(tbl, largelists, compress, denseunions, dictencode, dictencodenested)
                    if !isempty(cols.dictencodings)
                        for de in cols.dictencodings
                            dictsch = Tables.Schema((:col,), (eltype(de.data),))
                            existing = dictencodingvalues[de.id]
                            # get new de.data we haven't seen before for delta update
                            vals = setdiff(de.data, existing)
                            if !isempty(vals)
                                put!(msgs, makedictionarybatchmsg(dictsch, (col=vals,), de.id, true), i)
                                # add new de.data to existing set for future diffs
                                union!(existing, vals)
                            end
                        end
                    end
                    put!(msgs, makerecordbatchmsg(sch[], cols), i, true)
                catch e
                    showerror(stdout, e, catch_backtrace())
                    rethrow(e)
                end
            end
else
            @async begin
                try
                    cols = toarrowtable(tbl, largelists, compress, denseunions, dictencode, dictencodenested)
                    if !isempty(cols.dictencodings)
                        for de in cols.dictencodings
                            dictsch = Tables.Schema((:col,), (eltype(de.data),))
                            existing = dictencodingvalues[de.id]
                            # get new de.data we haven't seen before for delta update
                            vals = setdiff(de.data, existing)
                            put!(msgs, makedictionarybatchmsg(dictsch, (col=vals,), de.id, true), i)
                            # add new de.data to existing set for future diffs
                            union!(existing, vals)
                        end
                    end
                    put!(msgs, makerecordbatchmsg(sch[], cols), i, true)
                catch e
                    showerror(stdout, e, catch_backtrace())
                    rethrow(e)
                end
            end
end
        end
    end
    close(msgs)
    wait(tsk)
    # write empty message
    if !writetofile
        Base.write(io, Message(UInt8[], nothing, 0, true, false), blocks, sch)
    end
    if writetofile
        b = FlatBuffers.Builder(1024)
        schfoot = makeschema(b, sch[], firstcols[])
        if !isempty(blocks[1])
            N = length(blocks[1])
            Meta.footerStartRecordBatchesVector(b, N)
            for blk in Iterators.reverse(blocks[1])
                Meta.createBlock(b, blk.offset, blk.metaDataLength, blk.bodyLength)
            end
            recordbatches = FlatBuffers.endvector!(b, N)
        else
            recordbatches = FlatBuffers.UOffsetT(0)
        end
        if !isempty(blocks[2])
            N = length(blocks[2])
            Meta.footerStartDictionariesVector(b, N)
            for blk in Iterators.reverse(blocks[2])
                Meta.createBlock(b, blk.offset, blk.metaDataLength, blk.bodyLength)
            end
            dicts = FlatBuffers.endvector!(b, N)
        else
            dicts = FlatBuffers.UOffsetT(0)
        end
        Meta.footerStart(b)
        Meta.footerAddVersion(b, Meta.MetadataVersion.V4)
        Meta.footerAddSchema(b, schfoot)
        Meta.footerAddDictionaries(b, dicts)
        Meta.footerAddRecordBatches(b, recordbatches)
        foot = Meta.footerEnd(b)
        FlatBuffers.finish!(b, foot)
        footer = FlatBuffers.finishedbytes(b)
        Base.write(io, footer)
        Base.write(io, Int32(length(footer)))
        Base.write(io, "ARROW1")
    end
    return io
end

struct ToArrowTable
    sch::Tables.Schema
    cols::Vector{Any}
    metadata::Union{Nothing, Dict{String, String}}
    dictencodings::Vector{DictEncoding}
end

function toarrowtable(x, largelists, compress, denseunions, dictencode, dictencodenested)
    @debug 1 "converting input table to arrow formatted columns"
    cols = Tables.columns(x)
    meta = getmetadata(cols)
    sch = Tables.schema(cols)
    types = collect(sch.types)
    N = length(types)
    newcols = Vector{Any}(undef, N)
    newtypes = Vector{Type}(undef, N)
    dictencodings = DictEncoding[]
    Tables.eachcolumn(sch, cols) do col, i, nm
        newcol = toarrowvector(col, dictencodings; compression=compress, largelists=largelists, denseunions=denseunions, dictencode=dictencode, dictencodenested=dictencodenested)
        newtypes[i] = eltype(newcol)
        newcols[i] = newcol
    end
    # assign dict encoding ids
    for (i, de) in enumerate(dictencodings)
        de.id = i - 1
    end
    return ToArrowTable(Tables.Schema(sch.names, newtypes), newcols, meta, dictencodings)
end

Tables.columns(x::ToArrowTable) = x
Tables.rowcount(x::ToArrowTable) = length(x.cols) == 0 ? 0 : length(x.cols[1])
Tables.schema(x::ToArrowTable) = x.sch
Tables.columnnames(x::ToArrowTable) = x.sch.names
Tables.getcolumn(x::ToArrowTable, i::Int) = x.cols[i]

struct Message
    msgflatbuf
    columns
    bodylen
    isrecordbatch::Bool
    blockmsg::Bool
end

struct Block
    offset::Int64
    metaDataLength::Int32
    bodyLength::Int64
end

function Base.write(io::IO, msg::Message, blocks, sch)
    metalen = padding(length(msg.msgflatbuf))
    @debug 1 "writing message: metalen = $metalen, bodylen = $(msg.bodylen), isrecordbatch = $(msg.isrecordbatch)"
    if msg.blockmsg
        push!(blocks[msg.isrecordbatch ? 1 : 2], Block(position(io), metalen + 8, msg.bodylen))
    end
    # now write the final message spec out
    # continuation byte
    n = Base.write(io, 0xFFFFFFFF)
    # metadata length
    n += Base.write(io, Int32(metalen))
    # message flatbuffer
    n += Base.write(io, msg.msgflatbuf)
    n += writezeros(io, paddinglength(n))
    # message body
    if msg.columns !== nothing
        # write out buffers
        for col in Tables.Columns(msg.columns)
            writebuffer(io, col)
        end
    end
    return n
end

function makemessage(b, headerType, header, columns=nothing, bodylen=0)
    # write the message flatbuffer object
    Meta.messageStart(b)
    Meta.messageAddVersion(b, Meta.MetadataVersion.V5)
    Meta.messageAddHeaderType(b, headerType)
    Meta.messageAddHeader(b, header)
    Meta.messageAddBodyLength(b, Int64(bodylen))
    # Meta.messageAddCustomMetadata(b, meta)
    # Meta.messageStartCustomMetadataVector(b, num_meta_elems)
    msg = Meta.messageEnd(b)
    FlatBuffers.finish!(b, msg)
    return Message(FlatBuffers.finishedbytes(b), columns, bodylen, headerType == Meta.RecordBatch, headerType == Meta.RecordBatch || headerType == Meta.DictionaryBatch)
end

function makeschema(b, sch::Tables.Schema{names}, columns) where {names}
    # build Field objects
    N = length(names)
    fieldoffsets = [fieldoffset(b, names[i], columns.cols[i]) for i = 1:N]
    Meta.schemaStartFieldsVector(b, N)
    for off in Iterators.reverse(fieldoffsets)
        FlatBuffers.prependoffset!(b, off)
    end
    fields = FlatBuffers.endvector!(b, N)
    if columns.metadata !== nothing
        kvs = columns.metadata
        kvoffs = Vector{FlatBuffers.UOffsetT}(undef, length(kvs))
        for (i, (k, v)) in enumerate(kvs)
            koff = FlatBuffers.createstring!(b, String(k))
            voff = FlatBuffers.createstring!(b, String(v))
            Meta.keyValueStart(b)
            Meta.keyValueAddKey(b, koff)
            Meta.keyValueAddValue(b, voff)
            kvoffs[i] = Meta.keyValueEnd(b)
        end
        Meta.schemaStartCustomMetadataVector(b, length(kvs))
        for off in Iterators.reverse(kvoffs)
            FlatBuffers.prependoffset!(b, off)
        end
        meta = FlatBuffers.endvector!(b, length(kvs))
    else
        meta = FlatBuffers.UOffsetT(0)
    end
    # write schema object
    Meta.schemaStart(b)
    Meta.schemaAddEndianness(b, Meta.Endianness.Little)
    Meta.schemaAddFields(b, fields)
    Meta.schemaAddCustomMetadata(b, meta)
    return Meta.schemaEnd(b)
end

function makeschemamsg(sch::Tables.Schema, columns)
    @debug 1 "building schema message: sch = $sch"
    b = FlatBuffers.Builder(1024)
    schema = makeschema(b, sch, columns)
    return makemessage(b, Meta.Schema, schema)
end

function fieldoffset(b, name, col)
    nameoff = FlatBuffers.createstring!(b, String(name))
    T = eltype(col)
    nullable = T >: Missing
    # check for custom metadata
    if getmetadata(col) !== nothing
        kvs = getmetadata(col)
        kvoffs = Vector{FlatBuffers.UOffsetT}(undef, length(kvs))
        for (i, (k, v)) in enumerate(kvs)
            koff = FlatBuffers.createstring!(b, String(k))
            voff = FlatBuffers.createstring!(b, String(v))
            Meta.keyValueStart(b)
            Meta.keyValueAddKey(b, koff)
            Meta.keyValueAddValue(b, voff)
            kvoffs[i] = Meta.keyValueEnd(b)
        end
        Meta.fieldStartCustomMetadataVector(b, length(kvs))
        for off in Iterators.reverse(kvoffs)
            FlatBuffers.prependoffset!(b, off)
        end
        meta = FlatBuffers.endvector!(b, length(kvs))
    else
        meta = FlatBuffers.UOffsetT(0)
    end
    # build dictionary
    if isdictencoded(col)
        encodingtype = indtype(col)
        IT, inttype, _ = arrowtype(b, encodingtype)
        Meta.dictionaryEncodingStart(b)
        Meta.dictionaryEncodingAddId(b, Int64(getid(col)))
        Meta.dictionaryEncodingAddIndexType(b, inttype)
        # TODO: support isOrdered?
        Meta.dictionaryEncodingAddIsOrdered(b, false)
        dict = Meta.dictionaryEncodingEnd(b)
    else
        dict = FlatBuffers.UOffsetT(0)
    end
    type, typeoff, children = arrowtype(b, col)
    if children !== nothing
        Meta.fieldStartChildrenVector(b, length(children))
        for off in Iterators.reverse(children)
            FlatBuffers.prependoffset!(b, off)
        end
        children = FlatBuffers.endvector!(b, length(children))
    else
        Meta.fieldStartChildrenVector(b, 0)
        children = FlatBuffers.endvector!(b, 0)
    end
    # build field object
    if isdictencoded(col)
        @debug 1 "building field: name = $name, nullable = $nullable, T = $T, type = $type, inttype = $IT, dictionary id = $(getid(col))"
    else
        @debug 1 "building field: name = $name, nullable = $nullable, T = $T, type = $type"
    end
    Meta.fieldStart(b)
    Meta.fieldAddName(b, nameoff)
    Meta.fieldAddNullable(b, nullable)
    Meta.fieldAddTypeType(b, type)
    Meta.fieldAddType(b, typeoff)
    Meta.fieldAddDictionary(b, dict)
    Meta.fieldAddChildren(b, children)
    Meta.fieldAddCustomMetadata(b, meta)
    return Meta.fieldEnd(b)
end

struct FieldNode
    length::Int64
    null_count::Int64
end

struct Buffer
    offset::Int64
    length::Int64
end

function makerecordbatchmsg(sch::Tables.Schema{names, types}, columns) where {names, types}
    b = FlatBuffers.Builder(1024)
    recordbatch, bodylen = makerecordbatch(b, sch, columns)
    return makemessage(b, Meta.RecordBatch, recordbatch, columns, bodylen)
end

function makerecordbatch(b, sch::Tables.Schema{names, types}, columns) where {names, types}
    nrows = Tables.rowcount(columns)

    compress = nothing
    fieldnodes = FieldNode[]
    fieldbuffers = Buffer[]
    bufferoffset = 0
    for col in Tables.Columns(columns)
        if col isa Compressed
            compress = compressiontype(col)
        end
        bufferoffset = makenodesbuffers!(col, fieldnodes, fieldbuffers, bufferoffset)
    end
    @debug 1 "building record batch message: nrows = $nrows, sch = $sch, compress = $compress"

    # write field nodes objects
    FN = length(fieldnodes)
    Meta.recordBatchStartNodesVector(b, FN)
    for fn in Iterators.reverse(fieldnodes)
        Meta.createFieldNode(b, fn.length, fn.null_count)
    end
    nodes = FlatBuffers.endvector!(b, FN)

    # write buffer objects
    bodylen = 0
    BN = length(fieldbuffers)
    Meta.recordBatchStartBuffersVector(b, BN)
    for buf in Iterators.reverse(fieldbuffers)
        Meta.createBuffer(b, buf.offset, buf.length)
        bodylen += padding(buf.length)
    end
    buffers = FlatBuffers.endvector!(b, BN)

    # compression
    if compress !== nothing
        Meta.bodyCompressionStart(b)
        Meta.bodyCompressionAddCodec(b, compress)
        Meta.bodyCompressionAddMethod(b, Meta.BodyCompressionMethod.BUFFER)
        compression = Meta.bodyCompressionEnd(b)
    else
        compression = FlatBuffers.UOffsetT(0)
    end

    # write record batch object
    @debug 1 "built record batch message: nrows = $nrows, nodes = $fieldnodes, buffers = $fieldbuffers, compress = $compress, bodylen = $bodylen"
    Meta.recordBatchStart(b)
    Meta.recordBatchAddLength(b, Int64(nrows))
    Meta.recordBatchAddNodes(b, nodes)
    Meta.recordBatchAddBuffers(b, buffers)
    Meta.recordBatchAddCompression(b, compression)
    return Meta.recordBatchEnd(b), bodylen
end

function makedictionarybatchmsg(sch, columns, id, isdelta)
    @debug 1 "building dictionary message: id = $id, sch = $sch, isdelta = $isdelta"
    b = FlatBuffers.Builder(1024)
    recordbatch, bodylen = makerecordbatch(b, sch, columns)
    Meta.dictionaryBatchStart(b)
    Meta.dictionaryBatchAddId(b, Int64(id))
    Meta.dictionaryBatchAddData(b, recordbatch)
    Meta.dictionaryBatchAddIsDelta(b, isdelta)
    dictionarybatch = Meta.dictionaryBatchEnd(b)
    return makemessage(b, Meta.DictionaryBatch, dictionarybatch, columns, bodylen)
end

function makenodesbuffers!(col::MissingVector, fieldnodes, fieldbuffers, bufferoffset)
    push!(fieldnodes, FieldNode(length(col), length(col)))
    @debug 1 "made field node: nodeidx = $(length(fieldnodes)), col = $(typeof(col)), len = $(fieldnodes[end].length), nc = $(fieldnodes[end].null_count)"
    return bufferoffset
end

function writebuffer(io, col::MissingVector)
    return
end

function makenodesbuffers!(col::Primitive{T, S}, fieldnodes, fieldbuffers, bufferoffset) where {T, S}
    len = length(col)
    nc = nullcount(col)
    push!(fieldnodes, FieldNode(len, nc))
    @debug 1 "made field node: nodeidx = $(length(fieldnodes)), col = $(typeof(col)), len = $(fieldnodes[end].length), nc = $(fieldnodes[end].null_count)"
    # validity bitmap
    blen = nc == 0 ? 0 : bitpackedbytes(len)
    push!(fieldbuffers, Buffer(bufferoffset, blen))
    @debug 1 "made field buffer: bufferidx = $(length(fieldbuffers)), offset = $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded = $(padding(fieldbuffers[end].length))"
    # adjust buffer offset, make primitive array buffer
    bufferoffset += blen
    blen = len * sizeof(T)
    push!(fieldbuffers, Buffer(bufferoffset, blen))
    @debug 1 "made field buffer: bufferidx = $(length(fieldbuffers)), offset = $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded = $(padding(fieldbuffers[end].length))"
    return bufferoffset + padding(blen)
end

function writebitmap(io, col::ArrowVector)
    v = col.validity
    @debug 1 "writing validity bitmap: nc = $(v.nc), n = $(bitpackedbytes(v.ℓ))"
    v.nc == 0 && return 0
    Base.write(io, view(v.bytes, v.pos:(v.pos + bitpackedbytes(v.ℓ) - 1)))
end

function writebuffer(io, col::Primitive{T, S}) where {T, S}
    @debug 1 "writebuffer: col = $(typeof(col))"
    @debug 2 col
    writebitmap(io, col)
    n = writearray(io, S, col.data)
    @debug 1 "writing array: col = $(typeof(col.data)), n = $n, padded = $(padding(n))"
    writezeros(io, paddinglength(n))
    return
end

function makenodesbuffers!(col::Union{Map{T, O, A}, List{T, O, A}}, fieldnodes, fieldbuffers, bufferoffset) where {T, O, A}
    len = length(col)
    nc = nullcount(col)
    push!(fieldnodes, FieldNode(len, nc))
    @debug 1 "made field node: nodeidx = $(length(fieldnodes)), col = $(typeof(col)), len = $(fieldnodes[end].length), nc = $(fieldnodes[end].null_count)"
    # validity bitmap
    blen = nc == 0 ? 0 : bitpackedbytes(len)
    push!(fieldbuffers, Buffer(bufferoffset, blen))
    @debug 1 "made field buffer: bufferidx = $(length(fieldbuffers)), offset = $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded = $(padding(fieldbuffers[end].length))"
    # adjust buffer offset, make array buffer
    bufferoffset += blen
    blen = sizeof(O) * (len + 1)
    push!(fieldbuffers, Buffer(bufferoffset, blen))
    @debug 1 "made field buffer: bufferidx = $(length(fieldbuffers)), offset = $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded = $(padding(fieldbuffers[end].length))"
    bufferoffset += padding(blen)
    if eltype(A) == UInt8
        blen = length(col.data)
        push!(fieldbuffers, Buffer(bufferoffset, blen))
        @debug 1 "made field buffer: bufferidx = $(length(fieldbuffers)), offset = $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded = $(padding(fieldbuffers[end].length))"
        bufferoffset += padding(blen)
    else
        bufferoffset = makenodesbuffers!(col.data, fieldnodes, fieldbuffers, bufferoffset)
    end
    return bufferoffset
end

function writebuffer(io, col::Union{Map{T, O, A}, List{T, O, A}}) where {T, O, A}
    @debug 1 "writebuffer: col = $(typeof(col))"
    @debug 2 col
    writebitmap(io, col)
    # write offsets
    n = writearray(io, O, col.offsets.offsets)
    @debug 1 "writing array: col = $(typeof(col.offsets.offsets)), n = $n, padded = $(padding(n))"
    writezeros(io, paddinglength(n))
    # write values array
    if eltype(A) == UInt8
        n = writearray(io, UInt8, col.data)
        @debug 1 "writing array: col = $(typeof(col.data)), n = $n, padded = $(padding(n))"
        writezeros(io, paddinglength(n))
    else
        writebuffer(io, col.data)
    end
    return
end

function makenodesbuffers!(col::FixedSizeList{T, A}, fieldnodes, fieldbuffers, bufferoffset) where {T, A}
    len = length(col)
    nc = nullcount(col)
    push!(fieldnodes, FieldNode(len, nc))
    @debug 1 "made field node: nodeidx = $(length(fieldnodes)), col = $(typeof(col)), len = $(fieldnodes[end].length), nc = $(fieldnodes[end].null_count)"
    # validity bitmap
    blen = nc == 0 ? 0 : bitpackedbytes(len)
    push!(fieldbuffers, Buffer(bufferoffset, blen))
    @debug 1 "made field buffer: bufferidx = $(length(fieldbuffers)), offset = $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded = $(padding(fieldbuffers[end].length))"
    bufferoffset += blen
    if eltype(A) === UInt8
        blen = getN(Base.nonmissingtype(T)) * len
        push!(fieldbuffers, Buffer(bufferoffset, blen))
        @debug 1 "made field buffer: bufferidx = $(length(fieldbuffers)), offset = $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded = $(padding(fieldbuffers[end].length))"
        bufferoffset += padding(blen)
    else
        bufferoffset = makenodesbuffers!(col.data, fieldnodes, fieldbuffers, bufferoffset)
    end
    return bufferoffset
end

function writebuffer(io, col::FixedSizeList{T, A}) where {T, A}
    @debug 1 "writebuffer: col = $(typeof(col))"
    @debug 2 col
    writebitmap(io, col)
    # write values array
    if eltype(A) === UInt8
        n = writearray(io, UInt8, col.data)
        @debug 1 "writing array: col = $(typeof(col.data)), n = $n, padded = $(padding(n))"
        writezeros(io, paddinglength(n))
    else
        writebuffer(io, col.data)
    end
    return
end

function makenodesbuffers!(col::Struct{T}, fieldnodes, fieldbuffers, bufferoffset) where {T}
    len = length(col)
    nc = nullcount(col)
    push!(fieldnodes, FieldNode(len, nc))
    @debug 1 "made field node: nodeidx = $(length(fieldnodes)), col = $(typeof(col)), len = $(fieldnodes[end].length), nc = $(fieldnodes[end].null_count)"
    # validity bitmap
    blen = nc == 0 ? 0 : bitpackedbytes(len)
    push!(fieldbuffers, Buffer(bufferoffset, blen))
    @debug 1 "made field buffer: bufferidx = $(length(fieldbuffers)), offset = $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded = $(padding(fieldbuffers[end].length))"
    bufferoffset += blen
    for child in col.data
        bufferoffset = makenodesbuffers!(child, fieldnodes, fieldbuffers, bufferoffset)
    end
    return bufferoffset
end

function writebuffer(io, col::Struct)
    @debug 1 "writebuffer: col = $(typeof(col))"
    @debug 2 col
    writebitmap(io, col)
    # write values arrays
    for child in col.data
        writebuffer(io, child)
    end
    return
end

function makenodesbuffers!(col::Union{DenseUnion, SparseUnion}, fieldnodes, fieldbuffers, bufferoffset)
    len = length(col)
    nc = nullcount(col)
    push!(fieldnodes, FieldNode(len, nc))
    @debug 1 "made field node: nodeidx = $(length(fieldnodes)), col = $(typeof(col)), len = $(fieldnodes[end].length), nc = $(fieldnodes[end].null_count)"
    # typeIds buffer
    push!(fieldbuffers, Buffer(bufferoffset, len))
    @debug 1 "made field buffer: bufferidx = $(length(fieldbuffers)), offset = $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded = $(padding(fieldbuffers[end].length))"
    bufferoffset += padding(len)
    if col isa DenseUnion
        # offsets buffer
        blen = sizeof(Int32) * len
        push!(fieldbuffers, Buffer(bufferoffset, blen))
        @debug 1 "made field buffer: bufferidx = $(length(fieldbuffers)), offset = $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded = $(padding(fieldbuffers[end].length))"
        bufferoffset += padding(blen)
    end
    for child in col.data
        bufferoffset = makenodesbuffers!(child, fieldnodes, fieldbuffers, bufferoffset)
    end
    return bufferoffset
end

function writebuffer(io, col::Union{DenseUnion, SparseUnion})
    @debug 1 "writebuffer: col = $(typeof(col))"
    @debug 2 col
    # typeIds buffer
    n = writearray(io, UInt8, col.typeIds)
    @debug 1 "writing array: col = $(typeof(col.typeIds)), n = $n, padded = $(padding(n))"
    writezeros(io, paddinglength(n))
    if col isa DenseUnion
        n = writearray(io, Int32, col.offsets)
        @debug 1 "writing array: col = $(typeof(col.offsets)), n = $n, padded = $(padding(n))"
        writezeros(io, paddinglength(n))
    end
    for child in col.data
        writebuffer(io, child)
    end
    return
end

function makenodesbuffers!(col::DictEncoded{T, S}, fieldnodes, fieldbuffers, bufferoffset) where {T, S}
    len = length(col)
    nc = nullcount(col)
    push!(fieldnodes, FieldNode(len, nc))
    @debug 1 "made field node: nodeidx = $(length(fieldnodes)), col = $(typeof(col)), len = $(fieldnodes[end].length), nc = $(fieldnodes[end].null_count)"
    # validity bitmap
    blen = nc == 0 ? 0 : bitpackedbytes(len)
    push!(fieldbuffers, Buffer(bufferoffset, blen))
    @debug 1 "made field buffer: bufferidx = $(length(fieldbuffers)), offset = $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded = $(padding(fieldbuffers[end].length))"
    bufferoffset += blen
    # indices
    blen = sizeof(S) * len
    push!(fieldbuffers, Buffer(bufferoffset, blen))
    @debug 1 "made field buffer: bufferidx = $(length(fieldbuffers)), offset = $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded = $(padding(fieldbuffers[end].length))"
    bufferoffset += padding(blen)
    return bufferoffset
end

function writebuffer(io, col::DictEncoded)
    @debug 1 "writebuffer: col = $(typeof(col))"
    @debug 2 col
    writebitmap(io, col)
    # write indices
    n = writearray(io, col.indices)
    @debug 1 "writing array: col = $(typeof(col.indices)), n = $n, padded = $(padding(n))"
    writezeros(io, paddinglength(n))
    return
end

function makenodesbuffers!(col::Compressed, fieldnodes, fieldbuffers, bufferoffset)
    push!(fieldnodes, FieldNode(col.len, col.nullcount))
    @debug 1 "made field node: nodeidx = $(length(fieldnodes)), col = $(typeof(col)), len = $(fieldnodes[end].length), nc = $(fieldnodes[end].null_count)"
    for buffer in col.buffers
        blen = length(buffer.data) == 0 ? 0 : 8 + length(buffer.data)
        push!(fieldbuffers, Buffer(bufferoffset, blen))
        @debug 1 "made field buffer: bufferidx = $(length(fieldbuffers)), offset = $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded = $(padding(fieldbuffers[end].length))"
        bufferoffset += padding(blen)
    end
    for child in col.children
        bufferoffset = makenodesbuffers!(child, fieldnodes, fieldbuffers, bufferoffset)
    end
    return bufferoffset
end

function writearray(io, b::CompressedBuffer)
    if length(b.data) > 0
        n = Base.write(io, b.uncompressedlength)
        @debug 1 "writing compressed buffer: uncompressedlength = $(b.uncompressedlength), n = $(length(b.data))"
        @debug 2 b.data
        return n + Base.write(io, b.data)
    end
    return 0
end

function writebuffer(io, col::Compressed)
    @debug 1 "writebuffer: col = $(typeof(col))"
    @debug 2 col
    for buffer in col.buffers
        n = writearray(io, buffer)
        writezeros(io, paddinglength(n))
    end
    for child in col.children
        writebuffer(io, child)
    end
    return
end

max_stars_repo_name: StefanKarpinski/arrow

max_issues_repo_name: StefanKarpinski/arrow

max_forks_repo_name: StefanKarpinski/arrow


text:
@libcall arb_mul_2exp_si 2
@libcall arb_one 1
@libcall arb_pow_ui 2 prec=true
@libcall arb_fac_ui 1 prec=true
@libcall arb_div 3 prec=true
@libcall arb_abs 2 prec=true
@libcall arb_add_error 2
@libcall arb_add 3 prec=true
@libcall arb_sub 3 prec=true

ext: jl


text:
# Autogenerated wrapper script for IGLWrap_jll for x86_64-linux-gnu-cxx11
export libiglwrap

using boost_jll
using GMP_jll
using MPFR_jll
using OpenBLAS_jll
JLLWrappers.@generate_wrapper_header("IGLWrap")
JLLWrappers.@declare_library_product(libiglwrap, "libiglwrap.so")
function __init__()
    JLLWrappers.@generate_init_header(boost_jll, GMP_jll, MPFR_jll, OpenBLAS_jll)
    JLLWrappers.@init_library_product(
        libiglwrap,
        "lib/libiglwrap.so",
        RTLD_NOW | RTLD_LOCAL,
    )

    JLLWrappers.@generate_init_footer()
end  # __init__()


text:
export templateLeaves, templatePartition, templateRegion

function templateLeaves(alpha_leaf_prior::Float64,
                        priors::AbstractVector{<:Distribution},
                        likelihoods::AbstractVector{<:Distribution},
                        sstats::AbstractVector{<:AbstractSufficientStats},
                        N::Int, D::Int, K::Int)
    ids = map(k -> gensym("factorization"), 1:K)
    scopeVec = zeros(Bool, D)
    obsVec = zeros(Bool, N, K)
    priors_ = deepcopy(priors)
    likelihoods_ = deepcopy(likelihoods)
    sstats_ = mapreduce(k -> deepcopy(sstats), hcat, 1:K)

    return FactorizedDistributionGraphNode(
                                    gensym("fact"),
                                    scopeVec,
                                    obsVec,
                                    priors_,
                                    likelihoods_,
                                    sstats_
                                   )
end

function templateLeaves(alpha_leaf_prior::Float64,
                        priors::Vector{<:AbstractVector{<:Distribution}},
                        likelihoods::Vector{<:AbstractVector{<:Distribution}},
                        sstats::Vector{<:AbstractVector{<:AbstractSufficientStats}},
                        N::Int, D::Int, K::Int)
    ids = map(k -> gensym("factorization"), 1:K)
    scopeVec = zeros(Bool, D)
    obsVec = zeros(Int, N, K)
    weightpriors = map(d -> Dirichlet(length(likelihoods[d]), alpha_leaf_prior), 1:D)
    logweights = map(d -> log.(rand(weightpriors[d], K)), 1:D)
    priors_ = deepcopy(priors)
    likelihoods_ = deepcopy(likelihoods)
    sstats_ = map(d -> mapreduce(k -> deepcopy(sstats[d]), hcat, 1:K), 1:D)

    return FactorizedMixtureGraphNode(
                                    gensym("factmixture"),
                                    scopeVec,
                                    obsVec,
                                    weightpriors,
                                    logweights,
                                    priors_,
                                    likelihoods_,
                                    sstats_
                                   )
end

function templatePartition(alpha_region_prior::Float64,
                           alpha_partition_prior::Float64,
                           alpha_leaf_prior::Float64,
                           priors_leaf::AbstractVector,
                           likelihoods::AbstractVector,
                           sstats::AbstractVector,
                           N::Int, D::Int,
                           K_sum::Int, K_prod::Int,
                           J::Int, K::Int,
                           depth::Int, maxdepth::Int)

    children = if depth == maxdepth
        map(k -> templateLeaves(alpha_leaf_prior, priors_leaf, likelihoods, sstats, N, D, K), 1:K_prod)
    else
        map(k -> templateRegion(alpha_region_prior, alpha_partition_prior, alpha_leaf_prior,
                                priors_leaf, likelihoods, sstats, N, D, K_sum, K_prod, J, K, depth+1, maxdepth), 1:K_prod)
    end

    K_ = mapreduce(child -> length(child), *, children)
    scopeVec = zeros(Bool, D)
    obsVec = zeros(Bool, N, K_)
    prior = Dirichlet(K_prod, alpha_partition_prior)

    return PartitionGraphNode(
                              gensym("partition"),
                              scopeVec,
                              obsVec,
                              prior,
                              children
                             )
end

function templateRegion(alpha_region_prior::Float64,
                        alpha_partition_prior::Float64,
                        alpha_leaf_prior::Float64,
                        priors_leaf::AbstractVector,
                        likelihoods::AbstractVector,
                        sstats::AbstractVector,
                        N::Int, D::Int,
                        K_sum::Int, K_prod::Int,
                        J::Int, K::Int,
                        depth::Int, maxdepth::Int; root = false)

    K_ = root ? 1 : K_sum
    children = if depth == maxdepth
        map(k -> templateLeaves(alpha_leaf_prior, priors_leaf, likelihoods, sstats, N, D, K), 1:J)
    else
        map(k -> templatePartition(alpha_region_prior, alpha_partition_prior, alpha_leaf_prior,
                                   priors_leaf, likelihoods, sstats, N, D, K_sum, K_prod, J, K, depth+1, maxdepth), 1:J)
    end

    Ch = sum(length.(children))
    scopeVec = zeros(Bool, D)
    obsVec = zeros(Bool, N, K_)
    prior = Dirichlet(Ch, alpha_region_prior)
    logweights = convert(Matrix, reshape(mapreduce(_ -> rand(prior), hcat, 1:K_), Ch, K_))
    active = zeros(Bool, size(logweights)...)
    @assert size(logweights) == (Ch, K_)

    return RegionGraphNode(
                           gensym("region"),
                            scopeVec,
                            obsVec,
                            logweights,
                            active,
                            prior,
                            children
                          )

end

max_stars_repo_name: crinders/BayesianSumProductNetworks

max_issues_repo_name: crinders/BayesianSumProductNetworks

max_forks_repo_name: crinders/BayesianSumProductNetworks


text:
# interfaces for defining new compilers

# the definition of a new GPU compiler is typically split in two:
# - a generic compiler that lives in GPUCompiler.jl (e.g., emitting PTX, SPIR-V, etc)
# - a more specific version in a package that targets an environment (e.g. CUDA, ROCm, etc)
#
# the first level of customizability is found in the AbstractCompilerTarget hierarchy,
# with methods and interfaces that can only be implemented within GPUCompiler.jl.
#
# further customization should be put in a concrete instance of the AbstractCompilerParams
# type, and can be used to customize interfaces defined on CompilerJob.

## target

export AbstractCompilerTarget

# container for state handled by targets defined in GPUCompiler.jl

abstract type AbstractCompilerTarget end

source_code(@nospecialize(target::AbstractCompilerTarget)) = "text"

llvm_triple(@nospecialize(target::AbstractCompilerTarget)) = error("Not implemented")

function llvm_machine(@nospecialize(target::AbstractCompilerTarget))
    triple = llvm_triple(target)

    t = Target(triple=triple)

    tm = TargetMachine(t, triple)
    asm_verbosity!(tm, true)

    return tm
end

llvm_datalayout(target::AbstractCompilerTarget) = DataLayout(llvm_machine(target))

# the target's datalayout, with Julia's non-integral address spaces added to it
function julia_datalayout(@nospecialize(target::AbstractCompilerTarget))
    dl = llvm_datalayout(target)
    dl === nothing && return nothing
    DataLayout(string(dl) * "-ni:10:11:12:13")
end

## params

export AbstractCompilerParams

# container for state handled by external users of GPUCompiler.jl

abstract type AbstractCompilerParams end

## function specification

export FunctionSpec

# what we'll be compiling

struct FunctionSpec{F,TT}
    f::F
    tt::Type{TT}
    kernel::Bool
    name::Union{Nothing,String}
    world_age::UInt
end

# put the function and argument types in typevars
# so that we can access it from generated functions
# XXX: the default value of 0xffffffffffffffff is a hack, because we don't properly perform
#      world age intersection when querying the compilation cache. once we do, callers
#      should probably provide the world age of the calling code (!= the current world age)
#      so that querying the cache from, e.g. `cufuncton` is a fully static operation.
FunctionSpec(f, tt=Tuple{}, kernel=true, name=nothing, world_age=-1%UInt) =
    FunctionSpec{typeof(f),tt}(f, tt, kernel, name, world_age)

function Base.getproperty(@nospecialize(spec::FunctionSpec), sym::Symbol)
    if sym == :world
        # NOTE: this isn't used by the call to `hash` in `check_cache`,
        #       so we still use the raw world age there.
        age = spec.world_age
        return age == -1%UInt ? Base.get_world_counter() : age
    else
        return getfield(spec, sym)
    end
end

function signature(@nospecialize(spec::FunctionSpec))
    fn = something(spec.name, nameof(spec.f))
    args = join(spec.tt.parameters, ", ")
    return "$fn($(join(spec.tt.parameters, ", ")))"
end

function Base.show(io::IO, @nospecialize(spec::FunctionSpec))
    spec.kernel ? print(io, "kernel ") : print(io, "function ")
    print(io, signature(spec))
end

## job

export CompilerJob

# a specific invocation of the compiler, bundling everything needed to generate code

struct CompilerJob{T,P,F}
    target::T
    source::F
    params::P

    CompilerJob(target::AbstractCompilerTarget, source::FunctionSpec, params::AbstractCompilerParams) =
        new{typeof(target), typeof(params), typeof(source)}(target, source, params)
end

Base.similar(@nospecialize(job::CompilerJob), @nospecialize(source::FunctionSpec)) =
    CompilerJob(job.target, source, job.params)

function Base.show(io::IO, @nospecialize(job::CompilerJob{T})) where {T}
    print(io, "CompilerJob of ", job.source, " for ", T)
end

## interfaces and fallback definitions

# the Julia module to look up target-specific runtime functions in (this includes both
# target-specific functions from the GPU runtime library, like `malloc`, but also
# replacements functions for operations like `Base.sin`)
runtime_module(@nospecialize(job::CompilerJob)) = error("Not implemented")

# check if a function is an intrinsic that can assumed to be always available
isintrinsic(@nospecialize(job::CompilerJob), fn::String) = false

# provide a specific interpreter to use.
get_interpreter(@nospecialize(job::CompilerJob)) =
    GPUInterpreter(ci_cache(job), method_table(job), job.source.world)

# does this target support throwing Julia exceptions with jl_throw?
# if not, calls to throw will be replaced with calls to the GPU runtime
can_throw(@nospecialize(job::CompilerJob)) = false

# generate a string that represents the type of compilation, for selecting a compiled
# instance of the runtime library. this slug should encode everything that affects
# the generated code of this compiler job (with exception of the function source)
runtime_slug(@nospecialize(job::CompilerJob)) = error("Not implemented")

# early processing of the newly generated LLVM IR module
process_module!(@nospecialize(job::CompilerJob), mod::LLVM.Module) = return

# early processing of the newly identified LLVM kernel function
function process_entry!(@nospecialize(job::CompilerJob), mod::LLVM.Module,
                        entry::LLVM.Function)
    ctx = context(mod)

    if job.source.kernel
        # pass all bitstypes by value; by default Julia passes aggregates by reference
        # (this improves performance, and is mandated by certain back-ends like SPIR-V).
        args = classify_arguments(job, entry)
        for arg in args
            if arg.cc == BITS_REF
                push!(parameter_attributes(entry, arg.codegen.i), EnumAttribute("byval", 0, ctx))
            end
        end
    end

    return entry
end

# post-Julia optimization processing of the module
optimize_module!(@nospecialize(job::CompilerJob), mod::LLVM.Module) = return

# final processing of the IR module, right before validation and machine-code generation
finish_module!(@nospecialize(job::CompilerJob), mod::LLVM.Module) = return

add_lowering_passes!(@nospecialize(job::CompilerJob), pm::LLVM.PassManager) = return

link_libraries!(@nospecialize(job::CompilerJob), mod::LLVM.Module,
                undefined_fns::Vector{String}) = return

# whether pointer is a valid call target
valid_function_pointer(@nospecialize(job::CompilerJob), ptr::Ptr{Cvoid}) = false

# the codeinfo cache to use
ci_cache(@nospecialize(job::CompilerJob)) = GLOBAL_CI_CACHE

# the method table to use
method_table(@nospecialize(job::CompilerJob)) = GLOBAL_METHOD_TABLE

# how much debuginfo to emit
function llvm_debug_info(@nospecialize(job::CompilerJob))
    if Base.JLOptions().debug_level == 0
        LLVM.API.LLVMDebugEmissionKindNoDebug
    elseif Base.JLOptions().debug_level == 1
        LLVM.API.LLVMDebugEmissionKindLineTablesOnly
    elseif Base.JLOptions().debug_level >= 2
        LLVM.API.LLVMDebugEmissionKindFullDebug
    end
end


text:
# ranger_path = joinpath(@__DIR__,"../../")
# include(ranger_path * "ranger.jl")

# Run this template as:
#  julia --depwarn=error -- main.jl

include("../../ranger.jl")

include("game.jl")

using .RangerGame

println("Running Template Particles")

# using Debugger
RangerGame.go()

max_stars_repo_name: wdevore/Ranger-Julia-SDL

max_issues_repo_name: wdevore/Ranger-Julia-SDL

max_forks_repo_name: wdevore/Ranger-Julia-SDL


text:
#######################################################################################################################################################################################################
#
# Changes to this struct
# General
#     2022-Jan-14: Move the structure from Photosynthesis.jl, only P_A and P_O2 for now
#     2022-Jan-14: rename P_A to P_AIR, P_O2 to P_O₂
#     2022-Jan-14: add p_CO₂ to the structure
#     2022-Jan-24: fix documentation
#     2022-Mar-09: add t, p_H₂O, p_H₂O_sat, rh, and wind fields
#
#######################################################################################################################################################################################################
"""

$(TYPEDEF)

Structure that stores air layer information

# Fields

$(TYPEDFIELDS)

"""
mutable struct AirLayer{FT<:AbstractFloat}
    # parameters that do not change with time
    "Atmospheric pressure `[Pa]`"
    P_AIR::FT
    "O₂ partial pressure `[Pa]`"
    P_O₂::FT

    # prognostic variables that change with time
    "CO₂ partial pressure `[Pa]`"
    p_CO₂::FT
    "Temperature"
    t::FT

    # diagnodtic variables that change with time
    "H₂O partial pressure `[Pa]`"
    p_H₂O::FT
    "Saturated H₂O partial pressure `[Pa]`"
    p_H₂O_sat::FT
    "relative humidity"
    rh::FT
    "Wind speed `[m s⁻¹]`"
    wind::FT
end

#######################################################################################################################################################################################################
#
# Changes to this constructor
# General
#     2022-Jan-14: Move the structure from Photosynthesis.jl, only P_A and P_O2 for now
#     2022-Jan-24: add p_CO₂ to the constructors
#     2022-Mar-09: add t, p_H₂O, p_H₂O_sat, rh, and wind fields
#
#######################################################################################################################################################################################################
"""

    AirLayer{FT}() where {FT<:AbstractFloat}

Constructor for AirLayer

---
# Examples
```julia
air = AirLayer{Float64}();
```
"""
AirLayer{FT}() where {FT<:AbstractFloat} = (
    return AirLayer{FT}(
                P_ATM(FT),                                  # P_AIR
                P_ATM(FT) * 0.209,                          # P_O₂
                40,                                         # p_CO₂
                T_25(FT),                                   # t
                1500,                                       # p_H₂O
                saturation_vapor_pressure(T_25(FT)),        # p_H₂O_sat
                1500 / saturation_vapor_pressure(T_25(FT)), # rh
                1)                                          # wind
);


text:
#==============
MetaPhlAn Utils
==============#
const taxonlevels = (
    kingdom      = 1,
    phylum       = 2,
    class        = 3,
    order        = 4,
    family       = 5,
    genus        = 6,
    species      = 7,
    subspecies   = 8,
    unidentified = 0)

const shortlevels = (
    k = :kingdom,
    p = :phylum,
    c = :class,
    o = :order,
    f = :family,
    g = :genus,
    s = :species,
    t = :subspecies)

"""
    taxfilter!(df::DataFrame, level::Union{Int, Symbol}; keepunidentified::Bool)

Filter a MetaPhlAn table (as DataFrame) to a particular taxon level.
Levels may be given either as numbers or symbols:

- `1` = `:Kingdom`
- `2` = `:Phylum`
- `3` = `:Class`
- `4` = `:Order`
- `5` = `:Family`
- `6` = `:Genus`
- `7` = `:Species`
- `8` = `:Subspecies`

Taxon level is removed from resulting taxon string, eg.
`g__Bifidobacterium` becomes `Bifidobacterium`.

Set `keepunidentified` flag to `false` to remove `UNIDENTIFIED` rows.
"""
function taxfilter!(taxonomic_profile::AbstractDataFrame, level::Int=7; keepunidentified=true)
    in(level, collect(1:8)) || @error "$level not a valid taxonomic level" taxonlevels

    taxonomic_profile[!, 1] = last.(parsetaxa.(taxonomic_profile[!, 1]; throw=false))
    filter!(taxonomic_profile) do row
        tlev = taxonlevels[row[1][2]]
        keepunidentified ? in(tlev, (0, level)) : level == tlev
    end

    taxonomic_profile[!, 1] = map(t-> t[1], taxonomic_profile[!, 1])
    return taxonomic_profile
end

function taxfilter!(taxonomic_profile::AbstractDataFrame, level::Symbol; keepunidentified=true)
    in(level, keys(taxonlevels)) || @error "$level not a valid taxonomic level" taxonlevels
    taxfilter!(taxonomic_profile, taxonlevels[level]; keepunidentified=keepunidentified)
end

function taxfilter(taxonomic_profile::AbstractDataFrame, level::Int=7; keepunidentified=true)
    filt = deepcopy(taxonomic_profile)
    taxfilter!(filt, level; keepunidentified=keepunidentified)
    return filt
end

function taxfilter(taxonomic_profile::AbstractDataFrame, level::Symbol; keepunidentified=true)
    filt = deepcopy(taxonomic_profile)
    taxfilter!(filt, level; keepunidentified=keepunidentified)
    return filt
end

"""
    parsetaxon(taxstring::AbstractString, taxlevel::Union{Int, Symbol})

Levels may be given either as numbers or symbols:

- `1` = `:Kingdom`
- `2` = `:Phylum`
- `3` = `:Class`
- `4` = `:Order`
- `5` = `:Family`
- `6` = `:Genus`
- `7` = `:Species`
- `8` = `:Subspecies`
"""
function parsetaxon(taxstring::AbstractString, taxlevel::Int=7; throw=true)
    taxa = parsetaxa(taxstring, throw=throw)
    length(taxa) <= taxlevel || throw(ArgumentError("Taxonomy does not contain level $taxlevel"))

    return taxa[taxlevel]
end

parsetaxon(taxstring::AbstractString, taxlevel::Symbol) = parsetaxon(taxstring, taxonlevels[taxlevel])

function parsetaxa(taxstring::AbstractString; throw=true)
    taxa = split(taxstring, '|')
    return shortname.(taxa, throw=throw)
end

function shortname(taxon::AbstractString; throw=true)
    m = match(r"[kpcofgst]__(\w+)", taxon)

    if isnothing(m)
        throw ? throw(ArgumentError("Improperly formated taxon $taxon")) : return (string(taxon), :unidentified)
    end

    return (string(m.captures[1]), shortlevels[Symbol(first(taxon))])
end


text:
# BEGIN 1.2.1  deprecations
@deprecate get_energy_value(val::BatteryEMS) get_energy_surplus_cost(
    get_operation_cost(val),
)
@deprecate get_penalty_cost(val::BatteryEMS) get_energy_shortage_cost(
    get_operation_cost(val),
)

@deprecate set_energy_value!(val::BatteryEMS, v) set_energy_shortage_cost!(
    get_operation_cost(val),
    v,
)
@deprecate set_penalty_cost!(val::BatteryEMS, v) set_energy_surplus_cost!(
    get_operation_cost(val),
    v,
)

export get_energy_value
export get_penalty_cost
export set_energy_value!
export set_penalty_cost!

# END 1.2.1 deprecations


text:
module VortexHelperAutoFallingBlock
using ..Ahorn, Maple

@mapdef Entity "VortexHelper/AutoFallingBlock" AutoFallingBlock(
								x::Integer, y::Integer,
								width::Integer = 8,
								height::Integer = 8,
								tiletype::String = "3")

const placements = Ahorn.PlacementDict(
	"Auto-Falling Block (Vortex Helper)" => Ahorn.EntityPlacement(
		AutoFallingBlock,
		"rectangle",
		Dict{String, Any}(),
		Ahorn.tileEntityFinalizer
	)
)

Ahorn.editingOptions(entity::AutoFallingBlock) = Dict{String, Any}(
	"tiletype" => Ahorn.tiletypeEditingOptions()
)

Ahorn.minimumSize(entity::AutoFallingBlock) = 8, 8
Ahorn.resizable(entity::AutoFallingBlock) = true, true

Ahorn.selection(entity::AutoFallingBlock) = Ahorn.getEntityRectangle(entity)

Ahorn.renderAbs(ctx::Ahorn.Cairo.CairoContext, entity::AutoFallingBlock, room::Maple.Room) = Ahorn.drawTileEntity(ctx, room, entity)

end

max_stars_repo_name: catapillie/VortexHelper

max_issues_repo_name: catapillie/VortexHelper

max_forks_repo_name: catapillie/VortexHelper


text:
module FreeTypeAbstraction

using FreeType, GeometryTypes
using Compat
export newface
export renderface
export FontExtent
export kerning

immutable FontExtent{T}
    vertical_bearing    ::Vec{2, T}
    horizontal_bearing  ::Vec{2, T}

    advance             ::Vec{2, T}
    scale               ::Vec{2, T}
end
immutable FontFace
    name::String
end

import Base: ./, .*

.*{T, T2}(f::FontExtent{T}, scaling::Vec{2, T2}) = FontExtent(
    f.vertical_bearing * scaling[1],
    f.horizontal_bearing * scaling[2],
    f.advance .* scaling,
    f.scale .* scaling,
)
./{T, T2}(f::FontExtent{T}, scaling::Vec{2, T2}) = FontExtent(
    f.vertical_bearing ./ scaling,
    f.horizontal_bearing ./ scaling,
    f.advance ./ scaling,
    f.scale ./ scaling,
)

FontExtent(fontmetric::FreeType.FT_Glyph_Metrics, scale=64) = FontExtent(
    Vec{2, Float64}(fontmetric.vertBearingX, fontmetric.vertBearingY) / scale,
    Vec{2, Float64}(fontmetric.horiBearingX, fontmetric.horiBearingY) / scale,
    Vec{2, Float64}(fontmetric.horiAdvance, fontmetric.vertAdvance) / scale,
    Vec{2, Float64}(fontmetric.width, fontmetric.height) / scale
)

const FREE_FONT_LIBRARY = FT_Library[C_NULL]

function ft_init()
    global FREE_FONT_LIBRARY
    FREE_FONT_LIBRARY[1] != C_NULL && error("Freetype already initalized. init() called two times?")
    err = FT_Init_FreeType(FREE_FONT_LIBRARY)
    return err == 0
end

function ft_done()
    global FREE_FONT_LIBRARY
    FREE_FONT_LIBRARY[1] == C_NULL && error("Library == CNULL. FreeTypeAbstraction.done() called before init(), or done called two times?")
    err = FT_Done_FreeType(FREE_FONT_LIBRARY[1])
    FREE_FONT_LIBRARY[1] = C_NULL
    return err == 0
end

function newface(facename, faceindex::Real=0, ftlib=FREE_FONT_LIBRARY)
    face     = (FT_Face)[C_NULL]
    err     = FT_New_Face(ftlib[1], facename, Int32(faceindex), face)
    if err != 0
        error("FreeType could not load font $facename with error $err")
        return face[1]
    end
    face
end

setpixelsize(face, x, y) = setpixelsize(face, (x, y))

function setpixelsize(face, size)
    err = FT_Set_Pixel_Sizes(face[1], UInt32(size[1]), UInt32(size[2]))
    if err != 0
        error("Couldn't set the pixel size for font with error $err")
    end
end

function kerning(c1::Char, c2::Char, face::Array{Ptr{FreeType.FT_FaceRec},1}, divisor::Float32)
    i1 = FT_Get_Char_Index(face[], c1)
    i2 = FT_Get_Char_Index(face[], c2)
    kernVec = Array(FreeType.FT_Vector, 1)
    err = FT_Get_Kerning(face[], i1, i2, FreeType.FT_KERNING_DEFAULT, pointer(kernVec))
    if err != 0
        return zero(Vec{2, Float32})
    end
    return Vec{2, Float32}(kernVec[1].x / divisor, kernVec[1].y / divisor)
end

function loadchar(face, c::Char)
    err = FT_Load_Char(face[1], c, FT_LOAD_RENDER)
    @assert err == 0
end

function renderface(face, c::Char, pixelsize=(32,32))
    setpixelsize(face, pixelsize)
    faceRec = unsafe_load(face[1])
    loadchar(face, c)
    glyphRec    = unsafe_load(faceRec.glyph)
    @assert glyphRec.format == FreeType.FT_GLYPH_FORMAT_BITMAP
    return glyphbitmap(glyphRec.bitmap), FontExtent(glyphRec.metrics)
end

function getextent(face, c::Char, pixelsize)
    setpixelsize(face, pixelsize)
    faceRec = unsafe_load(face[1])
    loadchar(face, c)
    glyphRec = unsafe_load(faceRec.glyph)
    FontExtent(glyphRec.metrics)
end

function glyphbitmap(bmpRec::FreeType.FT_Bitmap)
    @assert bmpRec.pixel_mode == FreeType.FT_PIXEL_MODE_GRAY
    bmp = Array(UInt8, bmpRec.width, bmpRec.rows)
    row = bmpRec.buffer
    if bmpRec.pitch < 0
        row -= bmpRec.pitch * (rbmpRec.rows - 1)
    end

    for r = 1:bmpRec.rows
        srcArray = unsafe_wrap(Array, row, bmpRec.width)
        bmp[:, r] = srcArray
        row += bmpRec.pitch
    end
    return bmp
end

"""
Init and free c-lib
"""
function __init__()
    ft_init()
    atexit(ft_done)
end

end


text:
using Test
using Liblet

# Constructors

test_production_wrong_lhs() = Production(1, ["a"])

test_production_nonempty_lhs() = Production("",[""])

test_production_wrong_rhs() = Production("a",[1])

test_production_nonemptystr_rhs() = Production("a",["a","","c"])

test_production_nonempty_rhs() = Production("a",[])

test_production_inset() = begin
    P = Production("a", ["b", "c"])
    Q = Production("a", ("b", "c"))
    P == Q
end

test_production_aε() = Production("A", ["a", "ε"])

# Operators

test_production_unpack() = begin
    lhs, rhs = Production("a", ["b", "c"])
    ("a", ["b", "c"]) == (lhs, rhs)
end

test_production_totalorder() = Production("a", ["b"]) > Production("a", ["a"])

test_production_from_string_cf() = parseproduction("A B -> c", true)

# astype0

test_production_astype0() = Production(["a"], ["b"]) == astype0(Production("a", ["b"]))

# suchthat

test_production_such_that_lhs() = suchthat(left = "X")(Production("X", ["x"]))

test_production_such_that_rhs() = suchthat(right = "x")(Production("X", ["x"]))

test_production_such_that_rhs_len() = suchthat(rightlen = 2)(Production("X", ["x", "y"]))

test_production_such_that_rhs_is_suffix_of() = suchthat(right_is_suffix_of = ["a", "x"])(Production("X", ["x"]))

# Tests

function runproductiontests()
    @testset "production_tests" begin
        @test_throws ArgumentError test_production_wrong_lhs()
        @test_throws ArgumentError test_production_nonempty_lhs()
        @test_throws ArgumentError test_production_wrong_rhs()
        @test_throws ArgumentError test_production_nonemptystr_rhs()
        @test_throws ArgumentError test_production_nonempty_rhs()
        @test test_production_inset()
        @test_throws ArgumentError test_production_aε()

        @test test_production_unpack()
        @test test_production_totalorder()

        @test test_production_astype0()

        @test test_production_such_that_lhs()
        @test test_production_such_that_rhs()
        @test test_production_such_that_rhs_len()
        @test test_production_such_that_rhs_is_suffix_of()
    end
end


text:
struct CMDRedirect
    command::String
    arguments::Array{String,1}
end

function _call_with_redirect(cmd_in::CMDRedirect)
    command = `$(cmd_in.command)`
    arg = join(cmd_in.arguments, "\\n")
    redirect_printf = `printf $arg`

    run(pipeline(redirect_printf, stdout=pipeline(`$command`)))
end

function _xselect_cmd(mission_name::String, event_cl_path::String, out_basename::String="xselect_out.pi")
    if !isfile(event_cl_path)
        event_cl_path = string(event_cl_path, ".gz")
    end
    path_out = joinpath(dirname(event_cl_path), out_basename)

    session_name = split(tempname(), "/")[3]
    set_mission  = "set mission $(split(String(mission_name), "_")[1])"
    read_event   = [
        "read event",
        "$(dirname(event_cl_path))",
        "$(basename(event_cl_path))"
    ]
    extract_spec = "extract spectrum"
    save_spec    = "save spectrum $out_basename"

    if isfile(path_out)
        rm(path_out)
    end

    lines = vcat([
        session_name,
        set_mission,
        read_event,
        extract_spec,
        save_spec,
        "exit",
        "no"
    ]...)

    origina_dir = pwd()
    cd(dirname(event_cl_path))
    _call_with_redirect(CMDRedirect("xselect", lines))
    cd(origina_dir)
end

function _xselect_cmd(mission_name::String, obs_row::DataFrames.DataFrame)
    return _xselect_cmd(String(mission_name), obs_row[1, :event_cl][1])
end

function _grppha_cmd(path_xselect_spec::String, out_basename::String="grppha_out.pi")
    input    = path_xselect_spec
    path_out   = joinpath(dirname(path_xselect_spec), out_basename)
    respfile = "chkey respfile $(joinpath(ENV["CALDB"], "data/nicer/xti/cpf/rmf/nixtiref20170601v001.rmf"))"
    ancrfile = "chkey ancrfile $(joinpath(ENV["CALDB"], "data/nicer/xti/cpf/arf/nixtiaveonaxis20170601v002.arf"))"
    grouping = "group min 20"

    if isfile(path_out)
        rm(path_out)
    end

    lines = [
        input,
        path_out,
        respfile,
        ancrfile,
        grouping,
        "exit"
    ]

    _call_with_redirect(CMDRedirect("grppha", lines))
end

function _grppha_cmd(mission_name::String, obs_row::DataFrames.DataFrame)
    path_xselect_out = joinpath(dirname(obs_row[1, :event_cl][1]), "xselect_out.pi")
    return _grppha_cmd(path_xselect_out, "grppha_out.pi")
end

function _xspec_ldata_cmd(path_grppha_spec::String, e_min::Float64, e_max::Float64)
    plot_name = "plot_xspec_ldata_$(e_min)_$(e_max).gif"

    input    = "data $path_grppha_spec"
    plot_out = "cpd $plot_name/vgif"
    e_bound  = "ig 0.0-$e_min $e_max-**"
    set_e    = "setplot energy"
    plot_ld  = "plot ldata"

    lines = vcat([
        input,
        plot_out,
        e_bound,
        set_e,
        plot_ld,
        "exit"
    ]...)

    correct_path = "../../JAXTAM/images/$(e_min)_$(e_max)/espec/$plot_name"

    original_dir = pwd()
    cd(dirname(path_grppha_spec))
    _call_with_redirect(CMDRedirect("xspec", lines))
    rm(plot_name)
    mkpath(dirname(abspath(correct_path)))
    mv("$(plot_name)_2", correct_path; force=true)
    cd(original_dir)
    return(abspath(correct_path))
end

function _xspec_ldata_cmd(mission::Mission, obs_row::DataFrames.DataFrame, e_range::Tuple{Float64,Float64})
    path_grppha_out = joinpath(dirname(obs_row[1, :event_cl][1]), "grppha_out.pi")

    (e_min, e_max) = e_range

    plot_path = _xspec_ldata_cmd(path_grppha_out, e_min, e_max)
    plot_log  = JAXTAM._log_entry(; category=:images, e_range=(e_min, e_max), kind=:espec, file_name=basename(plot_path))
    JAXTAM._log_add(mission, obs_row, Dict("images"=>Dict((e_min, e_max)=>plot_log)))
end

function _xspec_eufspec_cmd(path_grppha_spec::String, e_min::Float64, e_max::Float64)
    plot_name = "plot_xspec_eufspec_$(e_min)_$(e_max).gif"

    input    = "data $path_grppha_spec"
    plot_out = "cpd $plot_name/vgif"
    e_bound  = "ig 0.0-$e_min $e_max-**"
    model    = [
        "model PowerLaw";
        repeat([""], 11); # Done to leave model parameters as defualts
        "newpar 1 0"
    ]
    set_e    = "setplot energy"
    plot_ld  = "plot eufspec"

    lines = vcat([
        input,
        plot_out,
        e_bound,
        model,
        set_e,
        plot_ld,
        "exit"
    ]...)

    correct_path = "../../JAXTAM/images/$(e_min)_$(e_max)/espec/$plot_name"

    original_dir = pwd()
    cd(dirname(path_grppha_spec))
    _call_with_redirect(CMDRedirect("xspec", lines))
    rm(plot_name)
    mkpath(dirname(abspath(correct_path)))
    mv("$(plot_name)_2", correct_path; force=true)
    cd(original_dir)
    return(abspath(correct_path))
end

function _xspec_eufspec_cmd(mission_name::Union{String,Symbol}, obs_row::DataFrames.DataFrame)
    path_grppha_out = joinpath(dirname(obs_row[1, :event_cl][1]), "grppha_out.pi")

    mission_config = JAXTAM.config(Symbol(mission_name))
    (e_min, e_max) = (Float64(mission_config.good_energy_min), Float64(mission_config.good_energy_max))

    plot_path = _xspec_eufspec_cmd(path_grppha_out, e_min, e_max)
    plot_log  = JAXTAM._log_entry(; category=:images, e_range=(e_min, e_max), kind=:espec, file_name=basename(plot_path))
    JAXTAM._log_add(Symbol(mission_name), obs_row, Dict("images"=>Dict((e_min, e_max)=>plot_log)))
end

function _call_all_espec(mission_name::Union{String,Symbol}, obs_row::DataFrames.DataFrame)
    _xselect_cmd(mission_name, obs_row)

    _grppha_cmd(mission_name, obs_row)

    _xspec_ldata_cmd(mission_name, obs_row)

    _xspec_eufspec_cmd(mission_name, obs_row)
end


text:
type RandomSearcher <: Optimizer
  name::ASCIIString
  search_space::SearchSpace
  best
  best_fitness
  RandomSearcher(searchSpace) = new("Random Search", searchSpace, nothing, nothing)
end

function ask(rs::RandomSearcher)
  # Just randomly generate a new individual and return it with a dummy index
  # (since we do not have a population there are no indices).
  [(rand_individual(rs.search_space), 1)]
end

function tell!(rs::RandomSearcher, rankedCandidates)
  candidate, index, fitness = rankedCandidates[1]
  if rs.best == nothing || fitness < rs.best_fitness
    rs.best = candidate
    rs.best_fitness = fitness
    return 1
  else
    return 0
  end
end

function random_search(parameters)
  RandomSearcher(parameters[:SearchSpace])
end


text:
@testset "69.sqrtx.jl" begin
    @test my_sqrt(4) == mysqrt(4) == 2
    @test my_sqrt(8) == mysqrt(8) == 2
end


text:
using LinearAlgebra

#Givens from Problem Statement
#Shear Modulus(psi),diameter(in),length(in),torque(lb*in)
G13 = 9.8*10^6
G2 = 11.2*10^6
d13 = 1.5
d2 = 1
l13 = 2*12
l2 = 1.5*12
τ = 2400

# Polar Second Area Moment of a Cylinder (in^4)
J_cyl(d) = 0.5*pi*(d/2)^4
J13 = J_cyl(d13)
J2 = J_cyl(d2)

# Torsional Stiffness Function
k(G,J,l) = (G*J)/l

# k_local(Elastic Modulus,thickness,height,length,number of nodes,
# row/column one,row/column two, ==1 left fixed/ ==2 right fixed)
function k_local(G,J,l,i,n1,n2,fixed=0)
    k_global = zeros(i,i)
    kin = k(G,J,l)
    k_global[n1,n1] = kin
    k_global[n1,n2] = -kin
    k_global[n2,n1] = -kin
    k_global[n2,n2] = kin
    if fixed == 1
      adj = zeros(1,i)
      adj[n1] = 1
      k_global[n1,:] = adj
      k_global[:,n1] = adj
    elseif fixed == 2
      adj = zeros(1,i)
      adj[n2] = 1
      k_global[n2,:] = adj
      k_global[:,n2] = adj
    end
    return k_global
end

# Global for angular displacement calculation
k_1 = k_local(G13,J13,l13,4,1,2,1)
k_2 = k_local(G2,J2,l2,4,2,3)
k_3 = k_local(G13,J13,l13,4,3,4,2)
kG = k_1+k_2+k_3

#Global for Reaction calculation
k_11 = k_local(G13,J13,l13,4,1,2)
k_22 = k_local(G2,J2,l2,4,2,3)
k_33 = k_local(G13,J13,l13,4,3,4)
kGG = k_11+k_22+k_33

# Torque Vector
t = Float64[0 0 τ 0]
T = transpose(t)

# Angular Displacement Vector
θ = kG\T
println("Nodal Angular Displacement Vector (radians)")
display(θ)

# Moment Reaction Vector
τ_app = kGG*θ
println("Moment Reaction Vector (lb*in):")
display(τ_app)


text:
using Test
using OceanTurbulenceParameterEstimation
using Oceananigans
using Oceananigans.Models.HydrostaticFreeSurfaceModels: ColumnEnsembleSize
using Oceananigans.TurbulenceClosures: ConvectiveAdjustmentVerticalDiffusivity
using Suppressor

const CAVD = ConvectiveAdjustmentVerticalDiffusivity

@testset "Ensembles of models with different closures" begin
    Nz = 16
    Hz = 1
    grid = RectilinearGrid(size=Nz, z=(-10, 10), topology=(Flat, Flat, Bounded), halo=1)

    closures = [CAVD(background_κz=1.0) CAVD(background_κz=1.1)
                CAVD(background_κz=1.2) CAVD(background_κz=1.3)]

    @test size(closures) == (2, 2)
    @test closures[2, 1].background_κz == 1.2

    Δt = 0.01 * grid.Δzᵃᵃᶜ^2

    model_kwargs = (; tracers=:c, buoyancy=nothing, coriolis=nothing)
    simulation_kwargs = (; Δt, stop_iteration=100)

    models = [HydrostaticFreeSurfaceModel(; grid, closure=closures[i, j], model_kwargs...) for i=1:2, j=1:2]

    set_ic!(model) = set!(model, c = (x, y, z) -> exp(-z^2))

    for model in models
        set_ic!(model)
        simulation = Simulation(model; simulation_kwargs...)
        run!(simulation)
    end

    ensemble_size = ColumnEnsembleSize(Nz=Nz, ensemble=(2, 2), Hz=1)
    ensemble_grid = RectilinearGrid(size=ensemble_size, z=(-10, 10), topology=(Flat, Flat, Bounded), halo=1)

    @test size(ensemble_grid) == (2, 2, Nz)

    ensemble_model = HydrostaticFreeSurfaceModel(; grid=ensemble_grid, closure=closures, model_kwargs...)
    set_ic!(ensemble_model)

    @test size(parent(ensemble_model.tracers.c)) == (2, 2, Nz+2)

    ensemble_simulation = Simulation(ensemble_model; simulation_kwargs...)
    run!(ensemble_simulation)

    for i = 1:2, j = 1:2
        @info "Testing IsotropicDiffusivity ensemble member ($i, $j)..."
        @test parent(ensemble_model.tracers.c)[i, j, :] == parent(models[i, j].tracers.c)[1, 1, :]
    end
end

@testset "Ensembles of models with different Coriolis parameters" begin
    Nz = 2
    Hz = 1
    grid = RectilinearGrid(size=Nz, z=(-1, 0), topology=(Flat, Flat, Bounded), halo=1)

    coriolises = [FPlane(f=0.0) FPlane(f=0.5)
                  FPlane(f=1.0) FPlane(f=1.1)]

    Δt = 0.01

    @test size(coriolises) == (2, 2)
    @test coriolises[2, 1].f == 1.0

    model_kwargs = (; tracers=nothing, buoyancy=nothing, closure=nothing)
    simulation_kwargs = (; Δt, stop_iteration=100)

    models = [HydrostaticFreeSurfaceModel(; grid, coriolis=coriolises[i, j], model_kwargs...) for i=1:2, j=1:2]

    set_ic!(model) = set!(model, u=1)

    for model in models
        set_ic!(model)
        simulation = Simulation(model; simulation_kwargs...)
        run!(simulation)
    end

    ensemble_size = ColumnEnsembleSize(Nz=Nz, ensemble=(2, 2), Hz=1)
    ensemble_grid = RectilinearGrid(size=ensemble_size, z=(-1, 0), topology=(Flat, Flat, Bounded), halo=1)
    ensemble_model = HydrostaticFreeSurfaceModel(; grid=ensemble_grid, coriolis=coriolises, model_kwargs...)
    set_ic!(ensemble_model)
    ensemble_simulation = Simulation(ensemble_model; simulation_kwargs...)
    run!(ensemble_simulation)

    for i = 1:2, j = 1:2
        @info "Testing Coriolis ensemble member ($i, $j)..."
        @test parent(ensemble_model.velocities.u)[i, j, :] == parent(models[i, j].velocities.u)[1, 1, :]
        @test parent(ensemble_model.velocities.v)[i, j, :] == parent(models[i, j].velocities.v)[1, 1, :]
    end
end


text:
module Lyra

using Electron, DataValues

import IteratorInterfaceExtensions, TableTraits, IterableTables, JSON, VegaLite
using FilePaths

export LyraWindow, add!

app = nothing

mutable struct LyraWindow
    w::Window

    function LyraWindow()
        main_html_uri = abs(join(@__PATH__, p"../assets/lyra/index.html"))

        global app

        if app == nothing
            app = Application()
        end

        w = Window(app, main_html_uri, options=Dict("title" => "Lyra"))

        new(w)
    end
end

function _add!(v::LyraWindow, source::Pair{Symbol,<:Any})
    TableTraits.isiterabletable(source[2]) === false && error("'data' is not a table.")

    it = IteratorInterfaceExtensions.getiterator(source[2])

    data_dict = Dict{String,Any}()

    data_dict["values"] = [Dict{Symbol,Any}(c[1] => isa(c[2], DataValue) ? (isna(c[2]) ? nothing : get(c[2])) : c[2] for c in zip(keys(r), values(r))) for r in it]

    x = [Dict{Symbol,Any}(c[1] => isa(c[2], DataValue) ? (isna(c[2]) ? nothing : get(c[2])) : c[2] for c in zip(keys(r), values(r))) for r in it]

    # data = JSON.json(data_dict)

    data = JSON.json(x)

    data_as_string = JSON.json(data)

    code = """
    global.addData('$(source[1])', $data_as_string)
    """

    run(v.w, code)

    return nothing
end

function add!(v::LyraWindow, source::Pair{Symbol,<:Any}, sources::Pair{Symbol,<:Any}...)
    _add!(v, source)

    for s in sources
        _add!(v, s)
    end
end

function (l::LyraWindow)(source)
    _add!(l, :dataset => source)
    return l
end

function LyraWindow(source)
    l = LyraWindow()

    _add!(l, :dataset => source)

    return l
end

function LyraWindow(source::Pair{Symbol,<:Any}, sources::Pair{Symbol,<:Any}...)
    l = LyraWindow()

    _add!(l, source)

    for s in sources
        _add!(l, s)
    end

    return l
end

function Base.getindex(v::LyraWindow)
    code = "global.getVegaSpec()"

    content = run(v.w, code)

    return VegaLite.VGSpec(content)
end

function Base.close(v::LyraWindow)
    close(v.w)
end

end # module


text:
module Doubles

export Double, twosum_fast, twosum, twoprod, scale, abs
import Base: +, -, *, /, sqrt, abs, scale, ==, convert, promote_rule, show

FloatTypes = Union{Float32,Float64}
abstract AbstractDouble{T} <: Real
immutable Single{T<:FloatTypes} <: AbstractDouble{T}
    hi::T
end
immutable Double{T<:FloatTypes} <: AbstractDouble{T}
    hi::T
    lo::T
end
Double(x) = Single(x)
Double{T}(x::Tuple{T,T}) = Double(x...)
DoubleNorm(x,y) = Double(twosum_fast(x,y)) # normalize the double

### promotions and conversions ###

convert{T<:FloatTypes}(::Type{T}, x::Single) = x.hi
convert{T<:FloatTypes}(::Type{T}, x::Double) = x.hi + x.lo
convert(::Type{BigFloat}, x::Single) = big(x.hi)
convert(::Type{BigFloat}, x::Double) = big(x.hi) + big(x.lo)

# The following hack promotes the float types to AbstractDouble so that
# float types get properly converted to a Single type.
# We need this since we do not want to promote floats to Double since
# we want to dispatch to methods with the Single type for effiency reasons.
# Similar for the other types.

convert{T}(::Type{AbstractDouble{T}}, z::T)               = Single(z)
convert{T}(::Type{AbstractDouble{T}}, z::Type{Single{T}}) = z
convert{T}(::Type{AbstractDouble{T}}, z::Type{Double{T}}) = z

promote_rule{T}(::Type{Single{T}}, ::Type{T})         = AbstractDouble{T}
promote_rule{T}(::Type{Double{T}}, ::Type{T})         = AbstractDouble{T}
promote_rule{T}(::Type{Double{T}}, ::Type{Single{T}}) = AbstractDouble{T}

### utility functions

# the following are only used for non fma systems
@inline trunclo(x::Float64) = reinterpret(Float64, reinterpret(UInt64, x) & 0xffff_ffff_f800_0000) # clear lower 27 bits (leave upper 26 bits)
@inline trunclo(x::Float32) = reinterpret(Float32, reinterpret(UInt32, x) & 0xffff_f000) # clear lowest 12 bits (leave upper 12 bits)
@inline function splitprec(x::FloatTypes)
    hx = trunclo(x)
    hx, x-hx
end

### basic error free float arithmetic ###

# fast two-sum addition, if |x| ≥ |y|
@inline function twosum_fast{T<:FloatTypes}(x::T, y::T)
    r = x + y
    r, y + (x - r)
end

# two-sum addition
@inline function twosum{T<:FloatTypes}(x::T, y::T)
    r = x + y
    v = r - x
    r, (y - v) + (x - (r - v))
end

# two-product fma
@inline function _twoprod_fma{T<:FloatTypes}(x::T, y::T)
    r = x*y
    r, fma(x,y,-r)
end

# two-product non-fma
@inline function _twoprod{T<:FloatTypes}(x::T, y::T)
    hx, lx = splitprec(x)
    hy, ly = splitprec(y)
    z = x*y
    z, ((hx*hy-z) + lx*hy + hx*ly) + lx*ly
end

### Double arithmetic ###

## negation

@inline -(x::Double)  = Double(-x.hi, -x.lo)
@inline -(x::Single)  = Single(-x.hi)

## addition

@inline function +{T}(x::Double{T}, y::Double{T})
    r, s = twosum(x.hi, y.hi)
    DoubleNorm(r, s + x.lo + y.lo)
end

@inline function +{T}(x::Double{T}, y::Single{T})
    r, s = twosum(x.hi, y.hi)
    DoubleNorm(r, s + x.lo)
end
@inline +{T}(x::Single{T}, y::Double{T}{T}) = y + x

@inline +{T}(x::Single{T}, y::Single{T}) = Double(twosum(x.hi, y.hi))

## subtraction

@inline function -{T}(x::Double{T}, y::Double{T})
    r, s = twosum(x.hi, -y.hi)
    DoubleNorm(r, s + (x.lo - y.lo))
end

@inline function -{T}(x::Double{T}, y::Single{T})
    r, s = twosum(x.hi, -y.hi)
    DoubleNorm(r, s + x.lo)
end

@inline function -{T}(x::Single{T}, y::Double{T})
    r, s = twosum(x.hi, -y.hi)
    DoubleNorm(r, s + y.lo)
end

@inline -{T}(x::Single{T}, y::Single{T}) = Double(twosum(x.hi, -y.hi))

## multiplication

@inline function *{T}(x::Double{T}, y::Double{T})
    r, s = twoprod(x.hi, y.hi)
    DoubleNorm(r, s + x.hi*y.lo + x.lo*y.hi)
end

@inline function *{T}(x::Double{T}, y::Single{T})
    z0, z1 = twoprod(x.hi, y.hi)
    DoubleNorm(z0, z1 + x.lo)
end
@inline *{T}(x::Single{T}, y::Double{T}) = y*x

@inline *{T}(x::Single{T}, y::Single{T}) = Double(twoprod(x.hi, y.hi))

## division

# private two-product div fma helper
@inline function _pdiv_fma{T<:FloatTypes}(x::T, y::T)
    ry = 1/y
    r = x*ry
    r, fma(-r,y,x), ry
end

# private two-product div helper
@inline function _pdiv{T<:FloatTypes}(x::T, y::T)
    ry = 1/y
    r = x*ry
    hx, lx = splitprec(r)
    hy, ly = splitprec(y)
    r, ((-hx*hy+r*y) - lx*hy - hx*ly) - lx*ly, ry
end

@inline function /{T}(x::Double{T}, y::Double{T})
    r, s, ry = pdiv(x.hi, y.hi)
    DoubleNorm(r, (s + muladd(-r, y.lo, x.lo))*ry)
end

@inline function /{T}(x::Double{T}, y::Single{T})
    r, s, ry = pdiv(x.hi, y.hi)
    DoubleNorm(r, (s - x.lo)*ry)
end

@inline function /{T}(x::Single{T}, y::Double{T})
    r, s, ry = pdiv(x.hi, y.hi)
    DoubleNorm(r, muladd(-r, y.lo, s)*ry)
end

@inline function /{T}(x::Single{T}, y::Single{T})
    r, s, ry = pdiv(x.hi, y.hi)
    Double(r, s*ry)
end

## square root

# fast sqrt (no domain checking) make sure to handle errors in calling method
_sqrt{T<:FloatTypes}(x::T) = Base.box(T, Base.sqrt_llvm_fast(Base.unbox(T, x)))

# x is double, z is double
@inline function sqrt(x::Double)
    r = _sqrt(x.hi)
    DoubleNorm(r, (x.lo + fma(-r, r, x.hi))/(r+r))
end

# x is single, z is double
@inline function sqrt(x::Single)
    r = _sqrt(x.hi)
    Double(r, fma(-r, r, x.hi)/(r+r))
end

### auxiliary

==(x::Double, y::Single) = x.hi == y.hi
==(x::Single, y::Double) = y == x

abs(x::Single) = Single(abs(x.hi))
abs(x::Double) = Double(abs(x.hi), abs(x.lo))

scale{T<:FloatTypes}(x::Double{T}, s::T) = Double(s*x.hi, s*x.lo)
scale{T<:FloatTypes}(s::T, x::Double{T}) = Double(s*x.hi, s*x.lo)
scale{T<:FloatTypes}(x::Single{T}, s::T) = Single(s*x.hi)
scale{T<:FloatTypes}(s::T, x::Single{T}) = Single(s*x.hi)

function show{T}(io::IO, x::Double{T})
    println(io, "Double{$T}")
    print(io, x.hi, ", ", x.lo)
end

# hack :P
function show{T}(io::IO, x::Single{T})
    println(io, "Double{$T}")
    print(io, x.hi)
end

# Determine if hardware FMA is available, should probably check with LLVM, see #9855.
# Checks if the `fma` function is fast for the floating point type `T`: typically is it a
# native instruction (`true`) or does it fall back on a software implementation (`false`).
function is_fma_fast end
for T in (Float32, Float64)
    @eval is_fma_fast(::Type{$T}) = $(muladd(nextfloat(one(T)),nextfloat(one(T)),-nextfloat(one(T),2)) != zero(T))
end
is_fma_fast() = is_fma_fast(Float64) && is_fma_fast(Float32)

if is_fma_fast()
    const twoprod = _twoprod_fma
    const pdiv = _pdiv_fma
else
    const twoprod = _twoprod
    const pdiv = _pdiv
end

end


text:
module JungleHelperUIImageTrigger

using ..Ahorn, Maple

@mapdef Trigger "JungleHelper/UIImageTrigger" UIImageTrigger(x::Integer, y::Integer, width::Integer=8, height::Integer=8, ImagePath::String="logo",Flag::String="",ImageX::Number=0.0,ImageY::Number=0.0,FadeIn::Number=1.0,FadeOut::Number=1.0)

const placements = Ahorn.PlacementDict(
    "UI Image Display (Jungle Helper)" => Ahorn.EntityPlacement(
       UIImageTrigger,
        "rectangle"
    )
)

end

max_stars_repo_name: Jackal-Celeste/JungleHelper

max_issues_repo_name: Jackal-Celeste/JungleHelper

max_forks_repo_name: Jackal-Celeste/JungleHelper


text:
#######################################
#
# Standard electrode sets
#
#######################################

EEG_64_10_20 = ["Fp1", "F3", "FC5", "C3", "CP5", "P3", "PO7", "Oz", "Fpz", "AFz", "F6", "FC4", "C2", "TP8", "P2", "P10", "AF7", "F5", "FC3", "C5", "CP3", "P5", "PO3", "POz", "Fp2", "Fz", "F8", "FC2", "C4", "CP6", "P4", "PO8", "AF3", "F7", "FC1", "T7", "CP1", "P7", "O1", "Pz", "AF8", "F2", "FT8", "FCz", "C6", "CP4", "P6", "PO4", "F1", "FT7", "C1", "TP7", "P1", "P9", "Iz", "CPz", "AF4", "F4", "FC6", "Cz", "T8", "CP2", "P8", "O2"]

EEG_Vanvooren_2014 = ["TP7", "P9", "P7", "P5", "P3", "P1", "PO7", "PO3", "O1", "P2", "P4", "P6", "P10", "TP8", "PO4", "PO8", "O2", "TP8"]

EEG_Vanvooren_2014_Left  = ["TP7", "P9", "P7", "P5", "P3", "P1", "PO7", "PO3", "O1"]

EEG_Vanvooren_2014_Right = ["P2", "P4", "P6", "P10", "TP8", "PO4", "PO8", "O2", "TP8"]


text:
# This file is a part of TypeDBClient.  License is MIT: https://github.com/Humans-of-Julia/TypeDBClient.jl/blob/main/LICENSE

#
# package typedb.client.api.answer;
#
# import typedb.client.api.concept.Concept;
#
# import javax.annotation.CheckReturnValue;
# import java.util.Collection;
# import java.util.Map;
#
# public interface ConceptMap {
#
#     @CheckReturnValue
#     Map<String, Concept> map();
#
#     @CheckReturnValue
#     Collection<Concept> concepts();
#
#     @CheckReturnValue
#     Concept get(String variable);
# }


text:
using FewShotAnomalyDetection
using UCI
using Flux
using MLDataPattern

hiddenDim = 32
latentDim = 3
numLayers = 3
nonlinearity = "relu"
layerType = "Dense"
β = 0.01

batchSize = 100
numBatches = 10000

function learn(train, test)
	svae = SVAEtwocaps(size(train[1], 1), hiddenDim, latentDim, numLayers, nonlinearity, layerType, :unit)
    learnRepresentation(data) = wloss(svae, data, β, (x, y) -> FewShotAnomalyDetection.mmd_imq(x, y, 1))
	opt = Flux.Optimise.ADAM(1e-5)
    cb = Flux.throttle(() -> println("SVAE: $(learnRepresentation(train[1]))"), 5)
	# there is a hack with RandomBatches because so far I can't manage to get them to work without the tuple - I have to find a different sampling iterator
    Flux.train!((x, y) -> learnRepresentation(x), Flux.params(svae), RandomBatches((train[1], zero(train[2])), size = batchSize, count = numBatches), opt, cb = cb)
	println("Train err: $(learnRepresentation(train[1])) vs test error: $(learnRepresentation(test[1]))")
end

dataset = "breast-cancer-wisconsin" # name of the UCI dataset. You can find the names in the e.g. in the LODA paper http://agents.fel.cvut.cz/stegodata/pdfs/Pev15-Loda.pdf - last page
dataset = "statlog-vehicle"

data, normal_labels, anomaly_labels = UCI.get_umap_data(dataset)
# This method iterates over all subdataset in a multiclass dataset or it only runs once for a single class dataset
subdatasets = UCI.create_multiclass(data, normal_labels, anomaly_labels)
train = []
test = []
for (subdata, class_label) in subdatasets
		println(dataset*" "*class_label)
		_X_tr, _y_tr, _X_tst, _y_tst = UCI.split_data(subdata, 0.8) #train/test split

		# dirty data normalization
		_X_tr .-= minimum(_X_tr)
		_X_tr ./= maximum(_X_tr)
		_X_tst .-= minimum(_X_tst)
		_X_tst ./= maximum(_X_tst)

		# no need to have it this way but we are used to it from legacy code
		train = (_X_tr, _y_tr)
		test = (_X_tst, _y_tst)

		# learn(train, test)
end

max_stars_repo_name: honzabim/FewShotAnomalyDetection

max_issues_repo_name: honzabim/FewShotAnomalyDetection

max_forks_repo_name: honzabim/FewShotAnomalyDetection


text:
using Fermi.DIIS

using TensorOperations
using LinearAlgebra
using Formatting
import Base: show

export UHF

abstract type UHFAlgorithm end

struct UHF <: AbstractHFWavefunction
    molecule::Molecule
    energy::Float64
    nocc::Int
    nvir::Int
    orbitals::UHFOrbitals
    e_conv::Float64
    d_conv::Float64
end

function UHF(x...)
    ref = Fermi.Options.get("reference")
    if ref != "uhf"
        throw(FermiException("reference ($ref) is not appropriate for uhf"))
    end
    if !any(i-> i isa UHFAlgorithm, x)
        UHF(x..., get_scf_alg())
    else
        throw(FermiException("invalid arguments for UHF method: $(x[1:end-1])"))
    end
end

struct UHFa <: UHFAlgorithm end
include("UHFa.jl")
include("UHFHelper.jl")

function string_repr(X::UHF)
    out = ""
    out = out*" ⇒ Fermi Unrestricted Hartree--Fock Wave function\n"
    out = out*" ⋅ Basis:                  $(X.orbitals.basis)\n"
    out = out*" ⋅ Energy:                 $(X.energy)\n"
    out = out*" ⋅ Occ. Spatial Orbitals:  $(X.nocc)\n"
    out = out*" ⋅ Vir. Spatial Orbitals:  $(X.nvir)\n"
    out = out*"Convergence: "
    out = out*"ΔE => $(format("{:1.2e}",abs(X.e_conv)))"
    out = out*" Dᵣₘₛ => $(format("{:1.2e}",abs(X.d_conv)))"
    return out
end

function show(io::IO, ::MIME"text/plain", X::UHF)
    print(string_repr(X))
end


text:
require(Pkg.dir("Knet/test/mnist.jl"))
using Knet
using MNIST: xtrn, ytrn, xtst, ytst

net = [Mmul(64), Bias(), Relu(),
       Mmul(10),  Bias(), XentLoss()]
setparam!(net; lr=0.01)

savenet("net0.jld", net)

@time for i=1:100
    train(net, xtrn, ytrn)
    println((i, accuracy(ytst, predict(net, xtst)),
                accuracy(ytrn, predict(net, xtrn))))
end

net = loadnet("net0.jld")
setparam!(net; lr=0.5)

@time for i=1:100
    train(net, xtrn, ytrn)
    println((i, accuracy(ytst, predict(net, xtst)),
                accuracy(ytrn, predict(net, xtrn))))
end

for h in (128, 256, 512, 1024)
    @show h
    net = [Mmul(h), Bias(), Relu(), Mmul(10),  Bias(), XentLoss()]
    setparam!(net; lr=0.5)
    @time for i=1:100
        train(net, xtrn, ytrn)
        println((i, accuracy(ytst, predict(net, xtst)),
                 accuracy(ytrn, predict(net, xtrn))))
    end
end

net = [Drop(0.2), Mmul(1024), Bias(), Relu(),
       Drop(0.5), Mmul(10),  Bias(), XentLoss()]
setparam!(net; lr=0.5)

@time for i=1:100
    train(net, xtrn, ytrn)
    println((i, accuracy(ytst, predict(net, xtst)),
             accuracy(ytrn, predict(net, xtrn))))
end

net = [Drop(0.2), Mmul(4096),  Bias(), Relu(),
       Drop(0.5), Mmul(4096), Bias(), Relu(),
       Drop(0.5), Mmul(10),   Bias(), XentLoss()]
setparam!(net; lr=0.5)

@time for i=1:100
    train(net, xtrn, ytrn)
    println((i, accuracy(ytst, predict(net, xtst)),
             accuracy(ytrn, predict(net, xtrn))))
end

net = [Conv(20,5), Bias(), Relu(), Pool(2),
       Conv(50,5), Bias(), Relu(), Pool(2),
       Mmul(500), Bias(), Relu(),
       Mmul(10), Bias(), XentLoss()]
setparam!(net; lr=0.1)

xtrn2 = reshape(xtrn, 28, 28, 1, size(xtrn, 2))
xtst2 = reshape(xtst, 28, 28, 1, size(xtst, 2))

@time for i=1:100
    train(net, xtrn2, ytrn)
    println((i, accuracy(ytst, predict(net, xtst2)),
             accuracy(ytrn, predict(net, xtrn2))))
end


text:
#=
@testset "test NGramHMM..." begin
        @time d = ZhuXian()
        @time m = KongYiji.NGramHMM(2, d, KongYiji.dir("usrdict1"), 0.)
        @time m2 = KongYiji.HMM(d, KongYiji.dir("usrdict1"), 0.)
        @test m.hpr == m2.hpr
        @test m.h2h == m2.h2h
        @test m.h2v == m2.h2v
        @test m.INF == m2.INF

        xs = "田不易独自一人站在树林里的僻静处，负手而立。"
        r = m(xs)
        r2 = m2(xs)
        println(r)
        println(r2)
end
  =#

@testset "Cross validating NGramHMM..." begin
        @time ctb = ChTreebank()

        nb = 10
        batches = KongYiji.foldbatch(ctb, nb)
        tbs2 = KongYiji.HmmScoreTable[]
        tbs3 = KongYiji.HmmScoreTable[]
        @showprogress 1 "Cross Validating ooHMM..." for (tr, te) in batches
                m3 = KongYiji.NGramHMM(3, tr, KongYiji.dir("usrdict1"), 0.)
                m2 = KongYiji.HMM(tr, KongYiji.dir("usrdict1"), 0.)
                x = KongYiji.rawsents(te)
                z = KongYiji.wordsents(te)
                @time y2 = m2(x)
                @time y3 = m3(x)
                t2 = KongYiji.HmmScoreTable(z, y2)
                t3 = KongYiji.HmmScoreTable(z, y3)
                @show t2
                @show t3
                push!(tbs2, t2)
                push!(tbs3, t3)
                #println(tbs[end])
        end
        println(sum(tbs2))
        println(sum(tbs3))
end

#=
@testset "Compare NGramHMM with HMM..." begin
        @time d = ZhuXian()
        @time m = KongYiji.NGramHMM(2, d, KongYiji.dir("usrdict1"), 0.)
        @time m2 = KongYiji.HMM(d, KongYiji.dir("usrdict1"), 0.)
        for sent in rawsents(d)
                y = m(sent)
                y2 = m2(sent)
                @test y == y2
                if y != y2; println(sent); end
        end
end

  =#


text:
__precompile__(false)

module FileManager

"""
TODO
"""
function waitForFileReady(path::String, func::Function; mode="r", tryCount=100, tryWait=0.1) #error when functions does not exists
	result=false
	for i = 1:tryCount
		#try reading file
		if stat(path).size > 0
			open(path, mode) do fp result=func(fp) end
			if result break end
		end
		Libc.systemsleep(tryWait) #wait
	end
	result
end

export waitForFileReady

"""
TODO
"""
function fileGetContents(path::String) #; tryCount=100, tryWait=0.1)
	content=nothing
	#waitForFileReady(path,(fp)->(content=read(fp, String); content != nothing);tryCount=tryCount,tryWait=tryWait)
  open(path, "r") do fp content=read(fp, String) end
  content
end

export fileGetContents

"""
TODO
"""
function fileReadLines(path::String, tryCount=100, tryWait=0.1)
	content=nothing
  open(path, "r") do fp
    for line in eachline(fp)
      if content == nothing content=line
      else content*="\n"*line
      end
    end
  end
  content
end

export fileReadLines

function findFiles(dir, target_ext="TXT")
  list=Dict{Symbol,Dict{Symbol,Any}}()
  list_paths=Dict{Symbol,Dict{Symbol,Any}}()
  target_ext=uppercase(target_ext)

  for f in readdir(dir)
    split=splitext(f)
    path=abspath(dir*f)
    name=split[1]
    ext=uppercase(replace(split[end],"."=>""))

    if !isfile(path) || ext != target_ext continue end

    file=Dict{Symbol,Any}()
    file[:file] = f
    file[:name] = name
    file[:ext] = ext
    file[:path] = path
    file[:content] = fileGetContents(path)
    file[:symbol]=key=Symbol(uppercase(name))

    list_paths[Symbol(path)] = list[key] = file
  end

  list, list_paths
end

export findFiles

end #FileManager

max_stars_repo_name: Gilga/JuliaOpenGL

max_issues_repo_name: Gilga/JuliaOpenGL

max_forks_repo_name: Gilga/JuliaOpenGL