text:
################################################################################
# Pooling in channels
################################################################################
function max_channel_pooling_forward{T}(input::Array{T,3}, output::Array{T,3}, mask::Array{Csize_t,3}, layer)
  spatial_dim, channels, num = size(input)
  pooled_chann = size(output, 2)

  for n = 1:num
    for pc = 1:pooled_chann
      cstart = (pc-1)*layer.stride - layer.pad[1] + 1
      cend   = min(cstart + layer.kernel - 1, channels)
      cstart = max(1, cstart)

      for s = 1:spatial_dim
        @inbounds output[s,pc,n] = input[s,cstart,n]
        @inbounds mask[s,pc,n] = cstart
      end

      for c = cstart+1:cend
        for s = 1:spatial_dim
          @inbounds maxval = output[s,pc,n]
          @inbounds val = input[s,c,n]
          if val > maxval
            @inbounds output[s,pc,n] = val
            @inbounds mask[s,pc,n] = c
          end
        end
      end
    end
  end
end

function mean_channel_pooling_forward{T}(input::Array{T,3}, output::Array{T,3}, integral::Array{T}, layer)
  spatial_dim_T, channels, num = size(input)
  pooled_chann = size(output, 2)
  one = convert(T, 1)
  neg_one = convert(T, -1)
  scale = 1/convert(T, layer.kernel)

  spatial_dim = spatial_dim_T * sizeof(T)
  fea_dim = spatial_dim * channels
  output_fea_dim = spatial_dim * pooled_chann

  for n = 1:num
    input_ptr = pointer(input) + fea_dim*(n-1)
    output_ptr = pointer(output) + output_fea_dim*(n-1)
    integral_ptr = pointer(integral)

    # compute integral image
    BLAS.blascopy!(spatial_dim_T, input_ptr, 1, integral_ptr, 1)
    for c = 2:channels
      BLAS.blascopy!(spatial_dim_T, input_ptr + (c-1)*spatial_dim, 1,
          integral_ptr + (c-1)*spatial_dim, 1)
      BLAS.axpy!(spatial_dim_T, one, integral_ptr + (c-2)*spatial_dim, 1,
          integral_ptr + (c-1)*spatial_dim, 1)
    end

    for pc = 1:pooled_chann
      cstart = (pc-1)*layer.stride - layer.pad[1] + 1
      cend   = min(cstart + layer.kernel - 1, channels)
      cstart = max(1, cstart)

      output_ptr_pc = output_ptr + (pc-1)*spatial_dim

      BLAS.blascopy!(spatial_dim_T, integral_ptr + (cend-1)*spatial_dim, 1,
          output_ptr_pc, 1)
      if cstart > 1
        BLAS.axpy!(spatial_dim_T, neg_one, integral_ptr + (cstart-2)*spatial_dim, 1,
            output_ptr_pc, 1)
      end
      BLAS.scal!(spatial_dim_T, scale, output_ptr_pc, 1)
    end
  end
end

function max_channel_pooling_backward{T}(input::Array{T,3}, output::Array{T,3}, mask::Array{Csize_t,3}, layer)
  spatial_dim, channels, num = size(input)
  pooled_chann = size(output, 2)

  fill!(input, 0)
  for n = 1:num
    for pc = 1:pooled_chann
      cstart = (pc-1)*layer.stride - layer.pad[1] + 1
      cend   = min(cstart + layer.kernel - 1, channels)
      cstart = max(1, cstart)

      for s = 1:spatial_dim
        @inbounds input[s,mask[s,pc,n],n] += output[s,pc,n]
      end
    end
  end
end

function mean_channel_pooling_backward{T}(input::Array{T,3}, output::Array{T,3}, layer)
  spatial_dim_T, channels, num = size(input)
  pooled_chann = size(output, 2)
  scale = 1/convert(T, layer.kernel)

  fill!(input, 0)

  spatial_dim = spatial_dim_T * sizeof(T)
  fea_dim = spatial_dim * channels
  output_fea_dim = spatial_dim * pooled_chann

  for n = 1:num
    input_ptr = pointer(input) + fea_dim*(n-1)
    output_ptr = pointer(output) + output_fea_dim*(n-1)

    for pc = 1:pooled_chann
      cstart = (pc-1)*layer.stride - layer.pad[1] + 1
      cend   = min(cstart + layer.kernel - 1, channels)
      cstart = max(1, cstart)
      output_ptr_pc = output_ptr + (pc-1)*spatial_dim

      for c = cstart:cend
        BLAS.axpy!(spatial_dim_T, scale, output_ptr_pc, 1,
            input_ptr + (c-1)*spatial_dim, 1)
      end
    end
  end
end


text:
# helper functions

function h2o_get(endpoint::AbstractString, args...; version = 3)
    url = "http://$(H2Oip):$(H2Oport)/$(version)/$(endpoint)"

    if !isempty(args)
        url = string(url, "?")
        for i in args
            url = string(url, i, "&")
        end
            url = url[1:end-1]
    end

    println("h2o_get: ", url, "\n")  # for debugging, remove later

    results = get(url)
    return results
end

function h2o_post(endpoint::AbstractString, args...; version = 3)
    url = "http://$(H2Oip):$(H2Oport)/$(version)/$(endpoint)"

    if !isempty(args)
        url = string(url, "?")
        for i in args
            url = string(url, i, "&")
        end
        url = url[1:end-1]
    end

    println("h2o_post: ", url, "\n")   # for debugging, remove later

    results = post(url)
    return results
end

ext: jl


text:
export Soln, SolvData, ExData

##
# Types for Examples
##

type ExData
  a::Function
  b::Function
  c::Function

  f::Function
  chi::Function
  g::Function
  phi::Function

  s::Function

  u::Function
  ux::Function
  ut::Function
  sp::Function

  γ::Function

  function ExData(a::Function, b::Function, c::Function, s::Function, u::Function, ux::Function, ut::Function, sp::Function, γf::Function)

    aux{T<:Real}(x::T, t::T)=a(x,t)*ux(x,t)
    auxx{T<:Real}(x::T, t::T)=(aux(x+gPDEDiffEst,t)-aux(x,t))/gPDEDiffEst

    f{T<:Real}(x::T, t::T)=-(auxx(x,t)+b(x,t)*ux(x,t)+c(x,t)*u(x,t)-ut(x,t))

    g{T<:Real}(t::T)=aux(zero(T), t)

    phi{T<:Real}(x::T)=u(x, zero(T))

    chi{T<:Real}(x::T, t::T)=a(x, t)*ux(x, t)+γf(x, t)*sp(t)

    return new(a,b,c,f,chi,g,phi,s,u,ux,ut,sp,γf)
  end
end

##
# Types for PDE Solver Routine
##
@lintpragma("Info me PDEtypes: Consider removing dx information from SolnA.")
type SolnA{T<:Real, T1<:Real}
  xgrid::Vector{T}
  tgrid::Vector{T}
  dx::Vector{T} # TODO: Consider removing dx information
  tau::T
  xbdy::Vector{T}
  soln::Matrix{T1}
end

SolnA(nSlice::Int, s::Function)=SolnA(linspace(0,1), linspace(0,1), linspace(0,1), 0.1, linspace(0,1), [one(Float) for x in linspace(0,1), t in linspace(0,1)])
typealias Soln SolnA

type SolvData
  verbose::Bool
  a::Function
  b::Function
  c::Function
  f::Function
  rhs::Function
  g::Function
  phi::Function
  s::Function
  grid::SolnA
  SolvData(verbose::Bool, a::Function, b::Function, c::Function, f::Function, rhs::Function, g::Function, phi::Function, s::Function, nSlice::Int) = new(verbose, a, b, c, f, rhs, g, phi, s, SolnA(nSlice, s))
end
SolvData(a::Function, b::Function, c::Function, f::Function, rhs::Function, g::Function, phi::Function, s::Function, nSlice::Int) = SolvData(gSetVerbose, a, b, c, f, rhs, g, phi, s, nSlice)

function SolvData(exin::ExData, nSlice::Int)
  rhs{T<:Real}(t::T) = exin.chi(exin.s(t), t) - exin.γ(exin.s(t), t) * exin.sp(t)
  return SolvData(gSetVerbose, exin.a, exin.b, exin.c, exin.f, rhs, exin.g, exin.phi, exin.s, nSlice)
end

# include("PDESolvDataHelpers.jl")

max_stars_repo_name: jgoldfar/isp-julia-mwe

max_issues_repo_name: jgoldfar/isp-julia-mwe

max_forks_repo_name: jgoldfar/isp-julia-mwe


text:
using ClassicalOrthogonalPolynomials, Test
import ClassicalOrthogonalPolynomials: OrthogonalPolynomialRatio, recurrencecoefficients

@testset "OrthogonalPolynomialRatio" begin
    P = Legendre()
    R = OrthogonalPolynomialRatio(P,0.1)
    @test P[0.1,1:10] ./ P[0.1,2:11] ≈ R[1:10]
    R = OrthogonalPolynomialRatio(P,-1)
    @test R[1:10] ≈ fill(-1,10)
end


text:
# This file is a part of Julia. License is MIT: https://julialang.org/license

# Operations with the file system (paths) ##

export
    cd,
    chmod,
    chown,
    cp,
    cptree,
    mkdir,
    mkpath,
    mktemp,
    mktempdir,
    mv,
    pwd,
    rename,
    readlink,
    readdir,
    rm,
    samefile,
    sendfile,
    symlink,
    tempdir,
    tempname,
    touch,
    unlink,
    walkdir

import .Base.RefValue

# get and set current directory

"""
    pwd() -> AbstractString

Get the current working directory.

# Examples
```julia-repl
julia> pwd()
"/home/JuliaUser"

julia> cd("/home/JuliaUser/Projects/julia")

julia> pwd()
"/home/JuliaUser/Projects/julia"
```
"""
function pwd()
    b = Vector{UInt8}(undef, 1024)
    len = RefValue{Csize_t}(length(b))
    uv_error(:getcwd, ccall(:uv_cwd, Cint, (Ptr{UInt8}, Ptr{Csize_t}), b, len))
    String(b[1:len[]])
end

"""
    cd(dir::AbstractString=homedir())

Set the current working directory.

# Examples
```julia-repl
julia> cd("/home/JuliaUser/Projects/julia")

julia> pwd()
"/home/JuliaUser/Projects/julia"

julia> cd()

julia> pwd()
"/home/JuliaUser"
```
"""
function cd(dir::AbstractString)
    uv_error("chdir $dir", ccall(:uv_chdir, Cint, (Cstring,), dir))
end
cd() = cd(homedir())

if Sys.iswindows()
    function cd(f::Function, dir::AbstractString)
        old = pwd()
        try
            cd(dir)
            f()
       finally
            cd(old)
        end
    end
else
    function cd(f::Function, dir::AbstractString)
        fd = ccall(:open, Int32, (Cstring, Int32), :., 0)
        systemerror(:open, fd == -1)
        try
            cd(dir)
            f()
        finally
            systemerror(:fchdir, ccall(:fchdir, Int32, (Int32,), fd) != 0)
            systemerror(:close, ccall(:close, Int32, (Int32,), fd) != 0)
        end
    end
end
"""
    cd(f::Function, dir::AbstractString=homedir())

Temporarily change the current working directory to `dir`, apply function `f` and
finally return to the original directory.

# Examples
```julia-repl
julia> pwd()
"/home/JuliaUser"

julia> cd(readdir, "/home/JuliaUser/Projects/julia")
34-element Array{String,1}:
 ".circleci"
 ".freebsdci.sh"
 ".git"
 ".gitattributes"
 ".github"
 ⋮
 "test"
 "ui"
 "usr"
 "usr-staging"

julia> pwd()
"/home/JuliaUser"
```
"""
cd(f::Function) = cd(f, homedir())

function checkmode(mode::Integer)
    if !(0 <= mode <= 511)
        throw(ArgumentError("Mode must be between 0 and 511 = 0o777"))
    end
    mode
end

"""
    mkdir(path::AbstractString; mode::Unsigned = 0o777)

Make a new directory with name `path` and permissions `mode`. `mode` defaults to `0o777`,
modified by the current file creation mask. This function never creates more than one
directory. If the directory already exists, or some intermediate directories do not exist,
this function throws an error. See [`mkpath`](@ref) for a function which creates all
required intermediate directories.
Return `path`.

# Examples
```julia-repl
julia> mkdir("testingdir")
"testingdir"

julia> cd("testingdir")

julia> pwd()
"/home/JuliaUser/testingdir"
```
"""
function mkdir(path::AbstractString; mode::Integer = 0o777)
    @static if Sys.iswindows()
        ret = ccall(:_wmkdir, Int32, (Cwstring,), path)
    else
        ret = ccall(:mkdir, Int32, (Cstring, UInt32), path, checkmode(mode))
    end
    systemerror(:mkdir, ret != 0; extrainfo=path)
    path
end

"""
    mkpath(path::AbstractString; mode::Unsigned = 0o777)

Create all directories in the given `path`, with permissions `mode`. `mode` defaults to
`0o777`, modified by the current file creation mask.
Return `path`.

# Examples
```julia-repl
julia> mkdir("testingdir")
"testingdir"

julia> cd("testingdir")

julia> pwd()
"/home/JuliaUser/testingdir"

julia> mkpath("my/test/dir")
"my/test/dir"

julia> readdir()
1-element Array{String,1}:
 "my"

julia> cd("my")

julia> readdir()
1-element Array{String,1}:
 "test"

julia> readdir("test")
1-element Array{String,1}:
 "dir"
```
"""
function mkpath(path::AbstractString; mode::Integer = 0o777)
    isdirpath(path) && (path = dirname(path))
    dir = dirname(path)
    (path == dir || isdir(path)) && return path
    mkpath(dir, mode = checkmode(mode))
    try
        mkdir(path, mode = mode)
    catch err
        # If there is a problem with making the directory, but the directory
        # does in fact exist, then ignore the error. Else re-throw it.
        if !isa(err, SystemError) || !isdir(path)
            rethrow()
        end
    end
    path
end

"""
    rm(path::AbstractString; force::Bool=false, recursive::Bool=false)

Delete the file, link, or empty directory at the given path. If `force=true` is passed, a
non-existing path is not treated as error. If `recursive=true` is passed and the path is a
directory, then all contents are removed recursively.

# Examples
```jldoctest
julia> mkpath("my/test/dir");

julia> rm("my", recursive=true)

julia> rm("this_file_does_not_exist", force=true)

julia> rm("this_file_does_not_exist")
ERROR: IOError: unlink: no such file or directory (ENOENT)
Stacktrace:
[...]
```
"""
function rm(path::AbstractString; force::Bool=false, recursive::Bool=false)
    if islink(path) || !isdir(path)
        try
            @static if Sys.iswindows()
                # is writable on windows actually means "is deletable"
                if (filemode(path) & 0o222) == 0
                    chmod(path, 0o777)
                end
            end
            unlink(path)
        catch err
            if force && isa(err, IOError) && err.code==Base.UV_ENOENT
                return
            end
            rethrow()
        end
    else
        if recursive
            for p in readdir(path)
                rm(joinpath(path, p), force=force, recursive=true)
            end
        end
        @static if Sys.iswindows()
            ret = ccall(:_wrmdir, Int32, (Cwstring,), path)
        else
            ret = ccall(:rmdir, Int32, (Cstring,), path)
        end
        systemerror(:rmdir, ret != 0, extrainfo=path)
    end
end

# The following use Unix command line facilities
function checkfor_mv_cp_cptree(src::AbstractString, dst::AbstractString, txt::AbstractString;
                                                          force::Bool=false)
    if ispath(dst)
        if force
            # Check for issue when: (src == dst) or when one is a link to the other
            # https://github.com/JuliaLang/julia/pull/11172#issuecomment-100391076
            if Base.samefile(src, dst)
                abs_src = islink(src) ? abspath(readlink(src)) : abspath(src)
                abs_dst = islink(dst) ? abspath(readlink(dst)) : abspath(dst)
                throw(ArgumentError(string("'src' and 'dst' refer to the same file/dir.",
                                           "This is not supported.\n  ",
                                           "`src` refers to: $(abs_src)\n  ",
                                           "`dst` refers to: $(abs_dst)\n")))
            end
            rm(dst; recursive=true)
        else
            throw(ArgumentError(string("'$dst' exists. `force=true` ",
                                       "is required to remove '$dst' before $(txt).")))
        end
    end
end

function cptree(src::AbstractString, dst::AbstractString; force::Bool=false,
                                                          follow_symlinks::Bool=false)
    isdir(src) || throw(ArgumentError("'$src' is not a directory. Use `cp(src, dst)`"))
    checkfor_mv_cp_cptree(src, dst, "copying"; force=force)
    mkdir(dst)
    for name in readdir(src)
        srcname = joinpath(src, name)
        if !follow_symlinks && islink(srcname)
            symlink(readlink(srcname), joinpath(dst, name))
        elseif isdir(srcname)
            cptree(srcname, joinpath(dst, name); force=force,
                                                 follow_symlinks=follow_symlinks)
        else
            sendfile(srcname, joinpath(dst, name))
        end
    end
end

"""
    cp(src::AbstractString, dst::AbstractString; force::Bool=false, follow_symlinks::Bool=false)

Copy the file, link, or directory from `src` to `dst`.
`force=true` will first remove an existing `dst`.

If `follow_symlinks=false`, and `src` is a symbolic link, `dst` will be created as a
symbolic link. If `follow_symlinks=true` and `src` is a symbolic link, `dst` will be a copy
of the file or directory `src` refers to.
Return `dst`.
"""
function cp(src::AbstractString, dst::AbstractString; force::Bool=false,
                                                      follow_symlinks::Bool=false)
    checkfor_mv_cp_cptree(src, dst, "copying"; force=force)
    if !follow_symlinks && islink(src)
        symlink(readlink(src), dst)
    elseif isdir(src)
        cptree(src, dst; force=force, follow_symlinks=follow_symlinks)
    else
        sendfile(src, dst)
    end
    dst
end

"""
    mv(src::AbstractString, dst::AbstractString; force::Bool=false)

Move the file, link, or directory from `src` to `dst`.
`force=true` will first remove an existing `dst`.
Return `dst`.

# Examples
```jldoctest; filter = r"Stacktrace:(\\n \\[[0-9]+\\].*)*"
julia> write("hello.txt", "world");

julia> mv("hello.txt", "goodbye.txt")
"goodbye.txt"

julia> "hello.txt" in readdir()
false

julia> readline("goodbye.txt")
"world"

julia> write("hello.txt", "world2");

julia> mv("hello.txt", "goodbye.txt")
ERROR: ArgumentError: 'goodbye.txt' exists. `force=true` is required to remove 'goodbye.txt' before moving.
Stacktrace:
 [1] #checkfor_mv_cp_cptree#10(::Bool, ::Function, ::String, ::String, ::String) at ./file.jl:293
[...]

julia> mv("hello.txt", "goodbye.txt", force=true)
"goodbye.txt"

julia> rm("goodbye.txt");

```
"""
function mv(src::AbstractString, dst::AbstractString; force::Bool=false)
    checkfor_mv_cp_cptree(src, dst, "moving"; force=force)
    rename(src, dst)
    dst
end

"""
    touch(path::AbstractString)

Update the last-modified timestamp on a file to the current time.
Return `path`.

# Examples
```julia-repl
julia> write("my_little_file", 2);

julia> mtime("my_little_file")
1.5273815391135583e9

julia> touch("my_little_file");

julia> mtime("my_little_file")
1.527381559163435e9
```

We can see the [`mtime`](@ref) has been modified by `touch`.
"""
function touch(path::AbstractString)
    f = open(path, JL_O_WRONLY | JL_O_CREAT, 0o0666)
    try
        if Sys.isunix()
            ret = ccall(:futimes, Cint, (Cint, Ptr{Cvoid}), fd(f), C_NULL)
            systemerror(:futimes, ret != 0, extrainfo=path)
        else
            t = time()
            futime(f,t,t)
        end
    finally
        close(f)
    end
    path
end

if Sys.iswindows()

function tempdir()
    temppath = Vector{UInt16}(undef, 32767)
    lentemppath = ccall(:GetTempPathW,stdcall,UInt32,(UInt32,Ptr{UInt16}),length(temppath),temppath)
    if lentemppath >= length(temppath) || lentemppath == 0
        error("GetTempPath failed: $(Libc.FormatMessage())")
    end
    resize!(temppath,lentemppath)
    return transcode(String, temppath)
end

const temp_prefix = cwstring("jl_")
function _win_tempname(temppath::AbstractString, uunique::UInt32)
    tempp = cwstring(temppath)
    tname = Vector{UInt16}(undef, 32767)
    uunique = ccall(:GetTempFileNameW,stdcall,UInt32,(Ptr{UInt16},Ptr{UInt16},UInt32,Ptr{UInt16}), tempp,temp_prefix,uunique,tname)
    lentname = something(findfirst(iszero,tname), 0)-1
    if uunique == 0 || lentname <= 0
        error("GetTempFileName failed: $(Libc.FormatMessage())")
    end
    resize!(tname,lentname)
    return transcode(String, tname)
end

function mktemp(parent=tempdir())
    filename = _win_tempname(parent, UInt32(0))
    return (filename, Base.open(filename, "r+"))
end

function mktempdir(parent=tempdir())
    seed::UInt32 = Libc.rand(UInt32)
    while true
        if (seed & typemax(UInt16)) == 0
            seed += 1
        end
        filename = _win_tempname(parent, seed)
        ret = ccall(:_wmkdir, Int32, (Ptr{UInt16},), cwstring(filename))
        if ret == 0
            return filename
        end
        systemerror(:mktempdir, Libc.errno()!=Libc.EEXIST)
        seed += 1
    end
end

function tempname()
    parent = tempdir()
    seed::UInt32 = rand(UInt32)
    while true
        if (seed & typemax(UInt16)) == 0
            seed += 1
        end
        filename = _win_tempname(parent, seed)
        if !ispath(filename)
            return filename
        end
        seed += 1
    end
end

else # !windows
# Obtain a temporary filename.
function tempname()
    d = get(ENV, "TMPDIR", C_NULL) # tempnam ignores TMPDIR on darwin
    p = ccall(:tempnam, Cstring, (Cstring,Cstring), d, :julia)
    systemerror(:tempnam, p == C_NULL)
    s = unsafe_string(p)
    Libc.free(p)
    return s
end

# Obtain a temporary directory's path.
tempdir() = dirname(tempname())

# Create and return the name of a temporary file along with an IOStream
function mktemp(parent=tempdir())
    b = joinpath(parent, "tmpXXXXXX")
    p = ccall(:mkstemp, Int32, (Cstring,), b) # modifies b
    systemerror(:mktemp, p == -1)
    return (b, fdio(p, true))
end

# Create and return the name of a temporary directory
function mktempdir(parent=tempdir())
    b = joinpath(parent, "tmpXXXXXX")
    p = ccall(:mkdtemp, Cstring, (Cstring,), b)
    systemerror(:mktempdir, p == C_NULL)
    return unsafe_string(p)
end

end # os-test

"""
    tempdir()

Obtain the path of a temporary directory (possibly shared with other processes).
"""
tempdir()

"""
    tempname()

Generate a temporary file path. This function only returns a path; no file is
created. The path is likely to be unique, but this cannot be guaranteed.

!!! warning

    This can lead to race conditions if another process obtains the same
    file name and creates the file before you are able to.
    Using [`mktemp()`](@ref) is recommended instead.
"""
tempname()

"""
    mktemp(parent=tempdir())

Return `(path, io)`, where `path` is the path of a new temporary file in `parent` and `io`
is an open file object for this path.
"""
mktemp(parent)

"""
    mktempdir(parent=tempdir())

Create a temporary directory in the `parent` directory and return its path.
If `parent` does not exist, throw an error.
"""
mktempdir(parent)

"""
    mktemp(f::Function, parent=tempdir())

Apply the function `f` to the result of [`mktemp(parent)`](@ref) and remove the
temporary file upon completion.
"""
function mktemp(fn::Function, parent=tempdir())
    (tmp_path, tmp_io) = mktemp(parent)
    try
        fn(tmp_path, tmp_io)
    finally
        # TODO: should we call GC.gc() first on error, to make it much more likely that `rm` succeeds?
        try
            close(tmp_io)
            rm(tmp_path)
        catch ex
            @error "mktemp cleanup" _group=:file exception=(ex, catch_backtrace())
        end
    end
end

"""
    mktempdir(f::Function, parent=tempdir())

Apply the function `f` to the result of [`mktempdir(parent)`](@ref) and remove the
temporary directory upon completion.
"""
function mktempdir(fn::Function, parent=tempdir())
    tmpdir = mktempdir(parent)
    try
        fn(tmpdir)
    finally
        # TODO: should we call GC.gc() first on error, to make it much more likely that `rm` succeeds?
        try
            rm(tmpdir, recursive=true)
        catch ex
            @error "mktempdir cleanup" _group=:file exception=(ex, catch_backtrace())
        end
    end
end

struct uv_dirent_t
    name::Ptr{UInt8}
    typ::Cint
end

"""
    readdir(dir::AbstractString=".") -> Vector{String}

Return the files and directories in the directory `dir` (or the current working directory if not given).

# Examples
```julia-repl
julia> readdir("/home/JuliaUser/Projects/julia")
34-element Array{String,1}:
 ".circleci"
 ".freebsdci.sh"
 ".git"
 ".gitattributes"
 ".github"
 ⋮
 "test"
 "ui"
 "usr"
 "usr-staging"
```
"""
function readdir(path::AbstractString)
    # Allocate space for uv_fs_t struct
    uv_readdir_req = zeros(UInt8, ccall(:jl_sizeof_uv_fs_t, Int32, ()))

    # defined in sys.c, to call uv_fs_readdir, which sets errno on error.
    err = ccall(:uv_fs_scandir, Int32, (Ptr{Cvoid}, Ptr{UInt8}, Cstring, Cint, Ptr{Cvoid}),
                eventloop(), uv_readdir_req, path, 0, C_NULL)
    err < 0 && throw(SystemError("unable to read directory $path", -err))
    #uv_error("unable to read directory $path", err)

    # iterate the listing into entries
    entries = String[]
    ent = Ref{uv_dirent_t}()
    while Base.UV_EOF != ccall(:uv_fs_scandir_next, Cint, (Ptr{Cvoid}, Ptr{uv_dirent_t}), uv_readdir_req, ent)
        push!(entries, unsafe_string(ent[].name))
    end

    # Clean up the request string
    ccall(:jl_uv_fs_req_cleanup, Cvoid, (Ptr{UInt8},), uv_readdir_req)

    return entries
end

readdir() = readdir(".")

"""
    walkdir(dir; topdown=true, follow_symlinks=false, onerror=throw)

Return an iterator that walks the directory tree of a directory.
The iterator returns a tuple containing `(rootpath, dirs, files)`.
The directory tree can be traversed top-down or bottom-up.
If `walkdir` encounters a [`SystemError`](@ref)
it will rethrow the error by default.
A custom error handling function can be provided through `onerror` keyword argument.
`onerror` is called with a `SystemError` as argument.

# Examples
```julia
for (root, dirs, files) in walkdir(".")
    println("Directories in \$root")
    for dir in dirs
        println(joinpath(root, dir)) # path to directories
    end
    println("Files in \$root")
    for file in files
        println(joinpath(root, file)) # path to files
    end
end
```

```julia-repl
julia> mkpath("my/test/dir");

julia> itr = walkdir("my");

julia> (root, dirs, files) = first(itr)
("my", ["test"], String[])

julia> (root, dirs, files) = first(itr)
("my/test", ["dir"], String[])

julia> (root, dirs, files) = first(itr)
("my/test/dir", String[], String[])
```
"""
function walkdir(root; topdown=true, follow_symlinks=false, onerror=throw)
    content = nothing
    try
        content = readdir(root)
    catch err
        isa(err, SystemError) || throw(err)
        onerror(err)
        # Need to return an empty closed channel to skip the current root folder
        chnl = Channel(0)
        close(chnl)
        return chnl
    end
    dirs = Vector{eltype(content)}()
    files = Vector{eltype(content)}()
    for name in content
        if isdir(joinpath(root, name))
            push!(dirs, name)
        else
            push!(files, name)
        end
    end

    function _it(chnl)
        if topdown
            put!(chnl, (root, dirs, files))
        end
        for dir in dirs
            path = joinpath(root,dir)
            if follow_symlinks || !islink(path)
                for (root_l, dirs_l, files_l) in walkdir(path, topdown=topdown, follow_symlinks=follow_symlinks, onerror=onerror)
                    put!(chnl, (root_l, dirs_l, files_l))
                end
            end
        end
        if !topdown
            put!(chnl, (root, dirs, files))
        end
    end

    return Channel(_it)
end

function unlink(p::AbstractString)
    err = ccall(:jl_fs_unlink, Int32, (Cstring,), p)
    uv_error("unlink", err)
    nothing
end

# For move command
function rename(src::AbstractString, dst::AbstractString)
    err = ccall(:jl_fs_rename, Int32, (Cstring, Cstring), src, dst)
    # on error, default to cp && rm
    if err < 0
        # force: is already done in the mv function
        cp(src, dst; force=false, follow_symlinks=false)
        rm(src; recursive=true)
    end
    nothing
end

function sendfile(src::AbstractString, dst::AbstractString)
    src_open = false
    dst_open = false
    local src_file, dst_file
    try
        src_file = open(src, JL_O_RDONLY)
        src_open = true
        dst_file = open(dst, JL_O_CREAT | JL_O_TRUNC | JL_O_WRONLY, filemode(src_file))
        dst_open = true

        bytes = filesize(stat(src_file))
        sendfile(dst_file, src_file, Int64(0), Int(bytes))
    finally
        if src_open && isopen(src_file)
            close(src_file)
        end
        if dst_open && isopen(dst_file)
            close(dst_file)
        end
    end
end

if Sys.iswindows()
    const UV_FS_SYMLINK_JUNCTION = 0x0002
end

"""
    symlink(target::AbstractString, link::AbstractString)

Creates a symbolic link to `target` with the name `link`.

!!! note
    This function raises an error under operating systems that do not support
    soft symbolic links, such as Windows XP.
"""
function symlink(p::AbstractString, np::AbstractString)
    @static if Sys.iswindows()
        if Sys.windows_version() < Sys.WINDOWS_VISTA_VER
            error("Windows XP does not support soft symlinks")
        end
    end
    flags = 0
    @static if Sys.iswindows()
        if isdir(p)
            flags |= UV_FS_SYMLINK_JUNCTION
            p = abspath(p)
        end
    end
    err = ccall(:jl_fs_symlink, Int32, (Cstring, Cstring, Cint), p, np, flags)
    @static if Sys.iswindows()
        if err < 0 && !isdir(p)
            @warn "On Windows, creating file symlinks requires Administrator privileges" maxlog=1 _group=:file
        end
    end
    uv_error("symlink",err)
end

"""
    readlink(path::AbstractString) -> AbstractString

Return the target location a symbolic link `path` points to.
"""
function readlink(path::AbstractString)
    req = Libc.malloc(_sizeof_uv_fs)
    try
        ret = ccall(:uv_fs_readlink, Int32,
            (Ptr{Cvoid}, Ptr{Cvoid}, Cstring, Ptr{Cvoid}),
            eventloop(), req, path, C_NULL)
        if ret < 0
            ccall(:uv_fs_req_cleanup, Cvoid, (Ptr{Cvoid},), req)
            uv_error("readlink", ret)
            @assert false
        end
        tgt = unsafe_string(ccall(:jl_uv_fs_t_ptr, Ptr{Cchar}, (Ptr{Cvoid},), req))
        ccall(:uv_fs_req_cleanup, Cvoid, (Ptr{Cvoid},), req)
        return tgt
    finally
        Libc.free(req)
    end
end

"""
    chmod(path::AbstractString, mode::Integer; recursive::Bool=false)

Change the permissions mode of `path` to `mode`. Only integer `mode`s (e.g. `0o777`) are
currently supported. If `recursive=true` and the path is a directory all permissions in
that directory will be recursively changed.
Return `path`.
"""
function chmod(path::AbstractString, mode::Integer; recursive::Bool=false)
    err = ccall(:jl_fs_chmod, Int32, (Cstring, Cint), path, mode)
    uv_error("chmod", err)
    if recursive && isdir(path)
        for p in readdir(path)
            if !islink(joinpath(path, p))
                chmod(joinpath(path, p), mode, recursive=true)
            end
        end
    end
    path
end

"""
    chown(path::AbstractString, owner::Integer, group::Integer=-1)

Change the owner and/or group of `path` to `owner` and/or `group`. If the value entered for `owner` or `group`
is `-1` the corresponding ID will not change. Only integer `owner`s and `group`s are currently supported.
Return `path`.
"""
function chown(path::AbstractString, owner::Integer, group::Integer=-1)
    err = ccall(:jl_fs_chown, Int32, (Cstring, Cint, Cint), path, owner, group)
    uv_error("chown",err)
    path
end

max_stars_repo_name: syntapy/julia

max_issues_repo_name: syntapy/julia

max_forks_repo_name: syntapy/julia


text:
function test_reshape_layer(backend::Backend, n_input, T, eps)
  println("-- Testing ReshapeLayer on $(typeof(backend)){$T}...")

  # make sure the prod of the first 3 dimensions are the same
  dims = [(3*2^isodd(i), 2*2^iseven(i), 4, abs(rand(Int))%5+3) for i = 1:n_input]

  input = [rand(T, dims[i]...) for i = 1:n_input]
  input_blob = Blob[make_blob(backend, x) for x in input]
  diff_blob  = Blob[make_blob(backend, x) for x in input]

  println("    > Setup")
  layer = ReshapeLayer(bottoms=Array{Symbol}(n_input), tops=Array{Symbol}(n_input),
      shape=(1,1,prod(dims[1][1:end-1])))
  state = setup(backend, layer, input_blob, diff_blob)

  println("    > Forward")
  forward(backend, state, input_blob)
  for i = 1:n_input
    got_output = zeros(T, size(input[i]))
    copy!(got_output, input_blob[i])
    @test all(abs.(got_output - input[i]) .< eps)
  end

  println("    > Backward")
  top_diff = [rand(T, size(input[i])) for i = 1:n_input]
  for i = 1:n_input
    copy!(diff_blob[i], top_diff[i])
  end
  backward(backend, state, input_blob, diff_blob)
  for i = 1:n_input
    got_grad = zeros(T, size(input[i]))
    copy!(got_grad, diff_blob[i])
    @test all(abs.(got_grad - top_diff[i]) .< eps)
  end

  shutdown(backend, state)
end
function test_reshape_layer(backend::Backend)
  test_reshape_layer(backend, 3, Float64, 1e-10)
  test_reshape_layer(backend, 3, Float32, 1e-4)
end

if test_cpu
  test_reshape_layer(backend_cpu)
end
if test_gpu
  test_reshape_layer(backend_gpu)
end


text:
import Base.show

"LMCLUS algorithm parameters"
mutable struct Parameters
    "Minimum dimension of the cluster"
    min_dim::Int
    "Maximum dimension of the cluster"
    max_dim::Int
    "Nominal number of resulting clusters"
    number_of_clusters::Int
    "Terminate algorithm upon founding specified number of clusters"
    stop_after_cluster::Int
    "Force to search clusters in high subspaces"
    force_max_dim::Bool
    "Fixed number of bins for the distance histogram"
    hist_bin_size::Int
    "Minimum number of bins for the distance histogram"
    min_bin_num::Int
    "Minimum cluster size (or noise size) in order to prevent generation of small clusters"
    min_cluster_size::Int
    "Separation best bound value is used for evaluating a goodness of separation characterized by a discriminability and a depth between modes of a distance histogram."
    best_bound::Float64
    "Sampling error bound determines a minimal number of samples required to correctly identify a linear manifold cluster."
    error_bound::Float64
    "Maximum histogram bin size"
    max_bin_portion::Float64
    "RNG seed"
    random_seed::Int64
    "Sampling heuristic (1-3)"
    sampling_heuristic::Int
    "Sampling factor used in one of sampling heuristics"
    sampling_factor::Float64
    "Enables a sampling for a distance histogram"
    histogram_sampling::Bool
    "Enables an alignment of a manifold cluster basis"
    basis_alignment::Bool
    "Enables a linear manifold cluster dimensionality detection"
    dim_adjustment::Bool
    "Ratio of manifold principal subspace variance"
    dim_adjustment_ratio::Float64
    "Enables the minimum description length heuristic for a complexity validation of a generated cluster"
    mdl::Bool
    "MDL model precision encoding constant"
    mdl_model_precision::Int
    "MDL data precision encoding constant"
    mdl_data_precision::Int
    "Quantization error of a bin size calculation"
    mdl_quant_error::Float64
    "Compression threshold value for discarding candidate clusters"
    mdl_compres_ratio::Float64
    "MDL algorithm"
    mdl_algo::DataType
    "Enable creation of bounded linear manifold clusters"
    bounded_cluster::Bool
    "Separation threshold algorithm"
    sep_algo::Type{<:Thresholding}

    Parameters(max_dim) = new(
        1,        # min_dim
        max_dim,  # max_dim
        10,       # number_of_clusters
        1000,     # stop_after_cluster
        false,    # force_max_dim
        0,        # hist_bin_size
        7,        # min_bin_num
        20,       # min_cluster_size
        1.0,      # best_bound
        0.0001,   # error_bound
        0.1,      # max_bin_portion
        0,        # random_seed
        3,        # sampling_heuristic
        0.01,     # sampling_factor
        false,    # histogram_sampling
        false,    # basis_alignment
        false,    # dim_adjustment
        0.99,     # dim_adjustment_ratio
        false,    # mdl
        32,       # mdl_model_precision
        16,       # mdl_data_precision
        0.001,    # mdl_quant_error
        1.05,     # mdl_compres_ratio
        MDL.OptimalQuant, # mdl_algo
        false,    # bounded_cluster
        Kittler   # sep_algo
    )
end

show(io::IO, p::Parameters) =
    print(io, """Linear Manifold Clustering parameters:
    Min dimension (min_dim): $(p.min_dim)
    Max dimension (max_dim): $(p.max_dim)
    Nominal number of clusters (number_of_clusters): $(p.number_of_clusters)
    Stop searching after number for clusters found (stop_after_cluster): $(p.stop_after_cluster)
    Force algorithm to search in higher dimensions (force_max_dim): $(p.force_max_dim)
    Minimum cluster size (min_cluster_size): $(p.min_cluster_size)
    Best bound (best_bound): $(p.best_bound)
    Error bound (error_bound): $(p.error_bound)
    Sample points for distance histogram (histogram_sampling): $(p.histogram_sampling)
    Histogram bins (hist_bin_size): $(p.hist_bin_size)
    Minimum number of histogram bins (min_bin_num): $(p.min_bin_num)
    Maximum histogram bin size (max_bin_portion): $(p.max_bin_portion)
    Sampling heuristic (sampling_heuristic): $(p.sampling_heuristic)
    Sampling factor (sampling_factor): $(p.sampling_factor)
    Random seed (random_seed): $(p.random_seed) (0 - random seed)
    Manifold cluster basis alignment (basis_alignment): $(p.basis_alignment)
    Manifold dimensionality adjustment (dim_adjustment): $(p.dim_adjustment)
    Ratio of manifold principal subspace variance (dim_adjustment_ratio): $(p.dim_adjustment_ratio)
    Use MDL heuristic (mdl): $(p.mdl)
    MDL model precision encoding (mdl_model_precision): $(p.mdl_model_precision)
    MDL data precision encoding (mdl_data_precision): $(p.mdl_data_precision)
    MDL quantizing error (mdl_quant_error): $(p.mdl_quant_error)
    MDL compression ratio threshold (mdl_compres_ratio): $(p.mdl_compres_ratio)
    MDL algorithm (mdl_algo): $(p.mdl_algo)
    Creation of bounded linear manifold clusters (bounded_cluster): $(p.bounded_cluster)
    Separation algorithm (sep_algo): $(p.sep_algo)""")


text:
struct GroupedDataFrame <: AbstractFuture
    data::Future
    length::Future
    parent::DataFrame
    groupcols::Future
    groupkwargs::Future

    # GroupedDataFrame() = new(Future(), Future(), Future())
    # GroupedDataFrame(gdf::GroupedDataFrame) =
    #     new(Future(), Future(gdf.nrows), Future(gdf.offset))
end

Banyan.convert(::Type{Future}, gdf::GroupedDataFrame) = gdf.data
Banyan.isview(gdf::GroupedDataFrame) = true
Banyan.sample_memory_usage(gdf::DataFrames.GroupedDataFrame) =
    total_memory_usage(gdf) - total_memory_usage(parent(gdf))

Base.length(gdf::GroupedDataFrame) = collect(gdf.length)
Base.size(gdf::GroupedDataFrame) = Tuple(length(gdf))
Base.ndims(gdf::GroupedDataFrame) = 1
DataFrames.groupcols(gdf::GroupedDataFrame) = groupcols(sample(gdf))
DataFrames.valuecols(gdf::GroupedDataFrame) = valuecols(sample(gdf))

# NOTE: For now we don't allow grouped dataframes to be copied since we are
# only supporting simple use-cases where you want to aggregate or transform
# or filter your grouped dataframe.

# NOTE: We don't need to implement any of the sample computation functions
# (that we implement in `df.jl`) because a `GroupedDataFrame` will never be
# assigned a `Grouped` PT. The sample computation functions are only used by
# the `Grouped` PT constructor. And we never want to assign the `Grouped` PT
# constructor to `GroupedDataFrame`s. `Blocked` will be sufficient.

# GroupedDataFrame creation

function DataFrames.groupby(df::DataFrame, cols; kwargs...)::GroupedDataFrame
    get(kwargs, :sort, true) || error("Groups cannot currently be ordered by how they originally appeared")

    gdf_data = Future()
    gdf_length = Future()
    cols = Future(cols)
    kwargs = Future(kwargs)
    gdf = GroupedDataFrame(Future(), gdf_length, df, cols, kwargs)

    # partition(df, Replicated())
    # partition(gdf, Replicated())
    # partition(gdf_length, Replicated())

    groupingkeys = Symbol.(names(sample(df), collect(cols)))

    partitioned_using_modules("DataFrames")
    partitioned_using() do
        keep_sample_rate(gdf, df)
    end

    partitioned_with() do
        pt(df, Grouped(df, by=groupingkeys, scaled_by_same_as=gdf))
        # TODO: Avoid circular dependency
        # TODO: Specify key for Blocked
        # TODO: Ensure that bangs in splitting functions in PF library are used
        # appropriately
        pt(gdf, Blocked(along=1) & ScaledBySame(as=df))
        pt(gdf_length, Reducing(quote + end)) # TODO: See if we can `using Banyan` on the cluster and avoid this
        pt(df, gdf, gdf_length, cols, kwargs, Replicated())
    end

    @partitioned df gdf gdf_length cols kwargs begin
        gdf = DataFrames.groupby(df, cols; kwargs...)
        gdf_length = DataFrames.length(gdf)
    end

    # allowedgroupingkeys = names(sample(df), compute(cols))
    # allowedgroupingkeys = get(kwargs, :sort, false) ? allowedgroupingkeys[1:1] : allowedgroupingkeys
    # union!(sample(df, :allowedgroupingkeys), allowedgroupingkeys)
    # setsample(gdf, :allowedgroupingkeys, allowedgroupingkeys)
    # for key in allowedgroupingkeys
    #     for balanced in [true, false]
    #         partition(df, Grouped(;key=key, balanced=balanced))
    #     end
    #     # Grouped computes keystatistics for key for df
    #     setsample(gdf, :keystatistics, key, sample(df, :keystatistics, key))
    # end

    # pt(gdf, Blocked(;dim=1), match=df, on=["balanced", "id"])
    # ptartition(gdf_length, Reducing(;reducer=+))
    # papt(df, gdf, gdf_length, cols, kwargs, Replicated())
    # # TODO: Ensure splitting/merging functions work for Blocked on GroupedDataFrame

    # mutated(gdf)
    # mutated(gdf_length)

    # @partitioned df gdf gdf_length cols kwargs begin
    #     gdf = groupby(df, cols; kwargs...)
    #     gdf_length = length(gdf)
    # end

    # gdf

    # # TODO: approximate -> sample and evaluate -> compute

    # # w.r.t. keys and axes, there are several things you need to know:
    # # - reuse of columns
    # # Create Future for result

    # # gdf = GroupedDataFrame()
    # # gdf_len = gdf.whole_len
    # # df_len = df.whole_len
    # # for (gpt, max_ngroups) in Grouped(gdf, )
    # # partition(gdf, Distributed(), parent=df)
    # # @partitioned df gdf begin end

    # # when merging a GroupedDataFrame which must be pseudogrouped,
    # # vcat the parents and the groupindices and modify the cat-ed parents
    # # to have a column for the parent index and the gorup iindex within that parent
    # # and then do a group-by on this
    # # for writing to disk, just be sure to put everything into a dataframe such that it
    # # can be read back and have a column that specifies how to group by

    gdf
end

# GroupedDataFrame column manipulation

function DataFrames.select(gdf::GroupedDataFrame, args...; kwargs...)
    get(kwargs, :ungroup, true) || throw(ArgumentError("Select/transform/combine/subset operations on grouped dataframes must produce dataframes"))
    get(kwargs, :copycols, true) || throw(ArgumentError("Select/transform/combine/subset operations on grouped dataframes cannot return a view"))
    get(kwargs, :keepkeys, true) || throw(ArgumentError("Select/transform/combine/subset operations on grouped dataframes must keep the grouping columns"))

    gdf_parent = gdf.parent
    groupcols = gdf.groupcols
    groupkwargs = gdf.groupkwargs
    res = Future()
    args = Future(args)
    kwargs = Future(kwargs)

    groupingkeys = Symbol.(names(sample(gdf_parent), collect(groupcols)))

    partitioned_using_modules("DataFrames")
    partitioned_using() do
        keep_sample_keys(if get(collect(kwargs), :keepkeys, true) groupingkeys else [] end, res, gdf_parent, drifted=true)
        keep_sample_rate(res, gdf_parent)
    end

    partitioned_with() do
        pt(gdf_parent, Grouped(gdf_parent, by=groupingkeys, scaled_by_same_as=res), match=res)
        pt(gdf, Blocked(along=1) & ScaledBySame(as=res))
        pt(res, ScaledBySame(as=gdf_parent))
        pt(gdf_parent, gdf, res, groupcols, groupkwargs, args, kwargs, Replicated())
    end

    # partition(gdf, Replicated())
    # partition(gdf_parent, Replicated())
    # partition(res, Replicated())

    # # TODO: Share sampled names if performance is impacted by repeatedly getting names

    # # allowedgroupingkeys = names(sample(gdf_parent), compute(groupcols))
    # # allowedgroupingkeys = get(collect(groupkwargs), :sort, false) ? allowedgroupingkeys[1:1] : allowedgroupingkeys
    # # union!(sample(gdf_parent, :allowedgroupingkeys), allowedgroupingkeys)
    # if get(collect(kwargs), :keepkeys, true)
    #     union!(sample(res, :allowedgroupingkeys), sample(gdf, :allowedgroupingkeys))
    # end
    # for key in sample(gdf_parent, :allowedgroupingkeys)
    #     setsample(res, :keystatistics, key, sample(gdf_parent, :keystatistics, key))
    #     for balanced in [true, false]
    #         partition(gdf_parent, Grouped(;key=key, balanced=balanced))
    #         if get(collect(kwargs), :keepkeys, true)
    #             partition(res, Partitioned(), match=gdf_parent)
    #         else
    #             partition(res, Blocked(dim=1), match=gdf_parent, on=["balanced", "id"])
    #         end
    #     end
    # end
    # partition(gdf, Blocked(;dim=1), match=gdf_parent, on=["balanced", "id"])

    # partition(groupcols, Replicated())
    # partition(groupkwargs, Replicated())
    # partition(args, Replicated())
    # partition(kwargs, Replicated())

    # # if kwargs[:ungroup]

    # # else
    # #     res = GroupedDataFrame(gdf)
    # #     res_nrows = res.nrows
    # #     partition(gdf, Pseudogrouped())
    # #     partition(args, Replicated())
    # #     partition(kwargs, Replicated())
    # #     @partitioned gdf res res_nrows args kwargs begin
    # #         res = select(gdf, args..., kwargs...)
    # #         res_nrows = length(gdf_nrows)
    # #     end
    # # end

    # mutated(res)

    @partitioned gdf gdf_parent groupcols groupkwargs args kwargs res begin
        if !(gdf isa DataFrames.GroupedDataFrame) || gdf.parent != gdf_parent
            gdf = DataFrames.groupby(gdf_parent, groupcols; groupkwargs...)
        end
        res = DataFrames.select(gdf, args...; kwargs...)
    end

    DataFrame(res, copy(gdf_parent.nrows))
end

function DataFrames.transform(gdf::GroupedDataFrame, args...; kwargs...)
    get(kwargs, :ungroup, true) || throw(ArgumentError("Select/transform/combine/subset operations on grouped dataframes must produce dataframes"))
    get(kwargs, :copycols, true) || throw(ArgumentError("Select/transform/combine/subset operations on grouped dataframes cannot return a view"))
    get(kwargs, :keepkeys, true) || throw(ArgumentError("Select/transform/combine/subset operations on grouped dataframes must keep the grouping columns"))

    gdf_parent = gdf.parent
    groupcols = gdf.groupcols
    groupkwargs = gdf.groupkwargs
    res = Future()
    args = Future(args)
    kwargs = Future(kwargs)

    # TODO: Put groupingkeys in GroupedDataFrame
    groupingkeys = Symbol.(names(sample(gdf_parent), collect(groupcols)))

    partitioned_using_modules("DataFrames")
    partitioned_using() do
        keep_sample_keys(
            get(collect(kwargs), :keepkeys, true) ? groupingkeys : [], res, gdf_parent,
            drifted=true
        )
        keep_sample_rate(res, gdf_parent)
    end

    # TODO: Maybe automatically infer sample properties (set with
    # `partitioned_using`) by looking at the actual annotations in
    # `partitioned_with`

    partitioned_with() do
        pt(gdf_parent, Grouped(gdf_parent, by=groupingkeys, scaled_by_same_as=res), match=res)
        pt(gdf, Blocked(along=1) & ScaledBySame(as=res))
        pt(res, ScaledBySame(as=gdf_parent))
        pt(gdf_parent, gdf, res, groupcols, groupkwargs, args, kwargs, Replicated())
    end

    @partitioned gdf gdf_parent groupcols groupkwargs args kwargs res begin
        if !(gdf isa DataFrames.GroupedDataFrame) || gdf.parent != gdf_parent
            gdf = DataFrames.groupby(gdf_parent, groupcols; groupkwargs...)
        end
        res = DataFrames.transform(gdf, args...; kwargs...)
    end

    DataFrame(res, copy(gdf_parent.nrows))
end

function DataFrames.combine(gdf::GroupedDataFrame, args...; kwargs...)
    get(kwargs, :ungroup, true) || throw(ArgumentError("Select/transform/combine/subset operations on grouped dataframes must produce dataframes"))
    get(kwargs, :copycols, true) || throw(ArgumentError("Select/transform/combine/subset operations on grouped dataframes cannot return a view"))
    get(kwargs, :keepkeys, true) || throw(ArgumentError("Select/transform/combine/subset operations on grouped dataframes must keep the grouping columns"))

    gdf_parent = gdf.parent
    groupcols = gdf.groupcols
    groupkwargs = gdf.groupkwargs
    res_nrows = Future()
    res = DataFrame(Future(), res_nrows)
    args = Future(args)
    kwargs = Future(kwargs)

    # TODO: Put groupingkeys in GroupedDataFrame
    groupingkeys = Symbol.(names(sample(gdf_parent), collect(groupcols)))

    partitioned_using_modules("DataFrames")
    partitioned_using() do
        keep_sample_keys(
            get(collect(kwargs), :keepkeys, true) ? groupingkeys : [], res, gdf_parent,
            drifted=true
        )
        keep_sample_rate(res, gdf_parent)
    end

    partitioned_with() do
        # TODO: If we want to support `keepkeys=false`, we need to make the
        # result be Blocked and `filtered_from` the input
        pts_for_filtering(gdf_parent, res, with=Grouped, by=groupingkeys)
        pt(gdf, Blocked(along=1) & ScaledBySame(as=gdf_parent))
        pt(res_nrows, Reducing(quote + end)) # TODO: Change to + if possible
        # pt(gdf_parent, res, gdf, res_nrows, groupcols, groupkwargs, args, kwargs, Replicated())
        pt(gdf_parent, res, gdf, res_nrows, groupcols, groupkwargs, args, kwargs, Replicated())
    end

    @partitioned gdf gdf_parent groupcols groupkwargs args kwargs res res_nrows begin
        if !(gdf isa DataFrames.GroupedDataFrame) || gdf.parent != gdf_parent
            gdf = DataFrames.groupby(gdf_parent, groupcols; groupkwargs...)
        end
        res = DataFrames.combine(gdf, args...; kwargs...)
        res_nrows = DataFrames.nrow(res)
    end

    res
end

function DataFrames.subset(gdf::GroupedDataFrame, args...; kwargs...)
    get(kwargs, :ungroup, true) || throw(ArgumentError("Select/transform/combine/subset operations on grouped dataframes must produce dataframes"))
    get(kwargs, :copycols, true) || throw(ArgumentError("Select/transform/combine/subset operations on grouped dataframes cannot return a view"))
    get(kwargs, :keepkeys, true) || throw(ArgumentError("Select/transform/combine/subset operations on grouped dataframes must keep the grouping columns"))

    gdf_parent = gdf.parent
    groupcols = gdf.groupcols
    groupkwargs = gdf.groupkwargs
    res_nrows = Future()
    res = DataFrame(Future(), res_nrows)
    args = Future(args)
    kwargs = Future(kwargs)

    # TODO: Put groupingkeys in GroupedDataFrame
    groupingkeys = Symbol.(names(sample(gdf_parent), collect(groupcols)))

    partitioned_using_modules("DataFrames")
    partitioned_using() do
        keep_sample_keys(
            get(collect(kwargs), :keepkeys, true) ? groupingkeys : [], res, gdf_parent,
            drifted=true
        )
        keep_sample_rate(res, gdf_parent)
    end

    partitioned_with() do
        pts_for_filtering(gdf_parent, res, with=Grouped, by=groupingkeys)
        pt(gdf, Blocked(along=1) & ScaledBySame(as=gdf_parent))
        pt(res_nrows, Reducing(quote (a, b) -> a .+ b end))
        pt(gdf_parent, res, gdf, res_nrows, groupcols, groupkwargs, args, kwargs, Replicated())
    end

    @partitioned gdf gdf_parent groupcols groupkwargs args kwargs res res_nrows begin
        if !(gdf isa DataFrames.GroupedDataFrame) || gdf.parent != gdf_parent
            gdf = DataFrames.groupby(gdf_parent, groupcols; groupkwargs...)
        end
        res = DataFrames.subset(gdf, args...; kwargs...)
        res_nrows = DataFrames.nrow(res)
    end

    res
end

# function transform(gdf::GroupedDataFrame)
#     get(kwargs, :ungroup, true) || throw(ArgumentError("Select/transform/combine/subset operations on grouped dataframes must produce dataframes"))
#     get(kwargs, :copycols, true) || throw(ArgumentError("Select/transform/combine/subset operations on grouped dataframes cannot return a view"))

#     gdf_parent = gdf.parent
#     groupcols = gdf.groupcols
#     groupkwargs = gdf.groupkwargs
#     res = Future()
#     args = Future(args)
#     kwargs = Future(kwargs)

#     partition(gdf, Replicated())
#     partition(gdf_parent, Replicated())
#     partition(res, Replicated())

#     if get(collect(kwargs), :keepkeys, true)
#         union!(sample(res, :allowedgroupingkeys), sample(gdf, :allowedgroupingkeys))
#     end
#     for key in sample(gdf_parent, :allowedgroupingkeys)
#         setsample(res, :keystatistics, key, sample(gdf_parent, :keystatistics, key))
#         for balanced in [true, false]
#             partition(gdf_parent, Grouped(;key=key, balanced=balanced))
#             if get(collect(kwargs), :keepkeys, true)
#                 partition(res, Partitioned(), match=gdf_parent)
#             else
#                 partition(res, Blocked(dim=1), match=gdf_parent, on=["balanced", "id"])
#             end
#         end
#     end
#     partition(gdf, Blocked(;dim=1), match=gdf_parent, on=["balanced", "id"])

#     partition(groupcols, Replicated())
#     partition(groupkwargs, Replicated())
#     partition(args, Replicated())
#     partition(kwargs, Replicated())

#     mutated(res)

#     @partitioned gdf gdf_parent groupcols groupkwargs args kwargs res begin
#         if gdf.parent != gdf_parent
#             gdf = groupby(gdf_parent, groupcols; groupkwargs...)
#         end
#         res = transform(gdf, args...; kwargs...)
#     end

#     res
# end

# function combine(gdf::GroupedDataFrame)
#     get(kwargs, :ungroup, true) || throw(ArgumentError("Select/transform/combine/subset operations on grouped dataframes must produce dataframes"))
#     get(kwargs, :copycols, true) || throw(ArgumentError("Select/transform/combine/subset operations on grouped dataframes cannot return a view"))

#     gdf_parent = gdf.parent
#     groupcols = gdf.groupcols
#     groupkwargs = gdf.groupkwargs
#     res = Future()
#     args = Future(args)
#     kwargs = Future(kwargs)

#     partition(gdf, Replicated())
#     partition(gdf_parent, Replicated())
#     partition(res, Replicated())

#     if get(collect(kwargs), :keepkeys, true)
#         union!(sample(res, :allowedgroupingkeys), sample(gdf, :allowedgroupingkeys))
#     end
#     for key in sample(gdf_parent, :allowedgroupingkeys)
#         for balanced in [true, false]
#             partition(gdf_parent, Grouped(;key=key, balanced=balanced))
#             if get(collect(kwargs), :keepkeys, true)
#                 partition(res, Grouped(key=key, balanced=false, id="*"), match=gdf_parent, on="divisions")
#             else
#                 partition(res, Blocked(dim=1, balanced=false, id="*"))
#             end
#         end
#     end
#     partition(gdf, Blocked(;dim=1), match=gdf_parent, on=["balanced", "id"])

#     partition(groupcols, Replicated())
#     partition(groupkwargs, Replicated())
#     partition(args, Replicated())
#     partition(kwargs, Replicated())

#     # TODO: Allow for putting multiple variables that share a PT in a call to partition

#     mutated(res)

#     @partitioned gdf gdf_parent groupcols groupkwargs args kwargs res begin
#         if gdf.parent != gdf_parent
#             gdf = groupby(gdf_parent, groupcols; groupkwargs...)
#         end
#         res = combine(gdf, args...; kwargs...)
#     end

#     res
# end

# # TODO: Implement filter using some framework for having references by keeping
# # track of the lineage of which code regions produced which

# function subset(gdf::GroupedDataFrame)
#     get(kwargs, :ungroup, true) || throw(ArgumentError("Select/transform/combine/subset operations on grouped dataframes must produce dataframes"))
#     get(kwargs, :copycols, true) || throw(ArgumentError("Select/transform/combine/subset operations on grouped dataframes cannot return a view"))

#     gdf_parent = gdf.parent
#     groupcols = gdf.groupcols
#     groupkwargs = gdf.groupkwargs
#     res = Future()
#     args = Future(args)
#     kwargs = Future(kwargs)

#     partition(gdf, Replicated())
#     partition(gdf_parent, Replicated())
#     partition(res, Replicated())

#     if get(collect(kwargs), :keepkeys, true)
#         union!(sample(res, :allowedgroupingkeys), sample(gdf, :allowedgroupingkeys))
#     end
#     for key in sample(gdf_parent, :allowedgroupingkeys)
#         for balanced in [true, false]
#             partition(gdf_parent, Grouped(;key=key, balanced=balanced))
#             if get(collect(kwargs), :keepkeys, true)
#                 partition(res, Grouped(key=key, balanced=false, id="*"), match=gdf_parent, on="divisions")
#             else
#                 partition(res, Blocked(dim=1, balanced=false, id="*"))
#             end
#         end
#     end
#     partition(gdf, Blocked(;dim=1), match=gdf_parent, on=["balanced", "id"])

#     partition(groupcols, Replicated())
#     partition(groupkwargs, Replicated())
#     partition(args, Replicated())
#     partition(kwargs, Replicated())

#     mutated(res)

#     @partitioned gdf gdf_parent groupcols groupkwargs args kwargs res begin
#         if gdf.parent != gdf_parent
#             gdf = groupby(gdf_parent, groupcols; groupkwargs...)
#         end
#         res = subset(gdf, args...; kwargs...)
#     end

#     res
# end

max_stars_repo_name: banyan-team/banyan-julia

max_issues_repo_name: banyan-team/banyan-julia

max_forks_repo_name: banyan-team/banyan-julia


text:
#
# Correctness Tests
#
if VERSION >= v"0.5.0-"
    using Base.Test
else
    using BaseTestNext
    const Test = BaseTestNext
end

using XMLconvert
using BioMedQuery.Processes
using BioMedQuery.Entrez
using BioMedQuery.Entrez.DB
using BioMedQuery.DBUtils
using BioMedQuery.UMLS
using MySQL
using SQLite
using DataStreams

my_tests = [("dbutils_sqlite.jl",   "       Testing: DBUtils SQLite"),
            ("dbutils_mysql.jl",    "       Testing: DBUtils MySQL"),
            ("entrez.jl",           "       Testing: ENTREZ"),
            ("entrez_types.jl",      "       Testing: Entrez Types"),
            ("ct.jl",               "       Testing: CLINICAL TRIALS"),
            ("umls.jl",             "       Testing: UMLS"),
            ("processes_mysql.jl",  "       Testing: Processes MySQL"),
            ("processes_sqlite.jl", "       Testing: Processes SQLite"),
            ("export_citations.jl", "       Testing: Export Citations")
            ]

println("Running tests:")

for (my_test, test_string) in my_tests
    println("-----------------------------------------")
    println("-----------------------------------------")
    println(test_string)
    println("-----------------------------------------")
    println("-----------------------------------------")

    include(my_test)
end


text:
# This file include the utils for starting an experiment
# Use ```include("utils.jl")``` in your experiments

import Base: run
using Requires
@require Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80" using Plots
include("BanditExpBase.jl")
using ...Algorithms
using ...Arms
using ...Algorithms: make_agents_with_k
# using ..BanditExpBase: run


text:
module LLVM

using Unicode
using Printf
using Libdl

## source code includes

include("base.jl")
include("version.jl")

const libllvm = Base.libllvm_path()
if libllvm === nothing
    error("""Cannot find the LLVM library loaded by Julia.

             Please use a version of Julia that has been built with USE_LLVM_SHLIB=1 (like the official binaries).
             If you are, please file an issue and attach the output of `Libdl.dllist()`.""")
end

module API
using CEnum
using ..LLVM
using ..LLVM: libllvm

llvm_version = if version() < v"12"
    "11"
elseif version().major == 12
    "12"
else
    "13"
end
libdir = joinpath(@__DIR__, "..", "lib")

if !isdir(libdir)
    error("""
    The LLVM API bindings for v$llvm_version do not exist.
    You might need a newer version of LLVM.jl for this version of Julia.""")
end
import LLVMExtra_jll: libLLVMExtra

include(joinpath(libdir, llvm_version, "libLLVM_h.jl"))
include(joinpath(libdir, "libLLVM_extra.jl"))
include(joinpath(libdir, "libLLVM_julia.jl"))
end # module API

# LLVM API wrappers
include("support.jl")
include("types.jl")
include("passregistry.jl")
include("init.jl")
include("core.jl")
include("linker.jl")
include("irbuilder.jl")
include("analysis.jl")
include("moduleprovider.jl")
include("pass.jl")
include("passmanager.jl")
include("execution.jl")
include("buffer.jl")
include("target.jl")
include("targetmachine.jl")
include("datalayout.jl")
include("ir.jl")
include("bitcode.jl")
include("transform.jl")
include("debuginfo.jl")
include("dibuilder.jl")
include("jitevents.jl")
include("utils.jl")

has_orc_v1() = v"8" <= LLVM.version() < v"12"
if has_orc_v1()
    include("orc.jl")
end

has_orc_v2() = v"12" <= LLVM.version()
if has_orc_v2()
    include("orcv2.jl")
end

include("interop.jl")

include("deprecated.jl")

## initialization

function __init__()
    # sanity checks
    @debug "Using LLVM $(version()) at $libllvm"
    if libllvm != Base.libllvm_path()
        @error "Mismatch between LLVM library used during precompilation ($libllvm) and the current run-time situation ($(Base.libllvm_path())). Please recompile the package."
    end
    if version() !== runtime_version()
        @error "Using a different version of LLVM ($(runtime_version())) than the one shipped with Julia ($(version())); this is unsupported"
    end

    _install_handlers()
    _install_handlers(GlobalContext())
end

end


text:
import Markdown

const _unexported_public_api = Any[
    ## ../docs/src/manual.md
    # Experimental
    channel_unordered,
    append_unordered!,
    ZipSource,
    GetIndex,
    SetIndex,
    Inject,
    ## ../docs/src/interface.md
    # Core interface for transducers
    Transducer,
    AbstractFilter,
    R_,
    inner,
    xform,
    start,
    next,
    complete,
    # Helpers for stateful transducers
    wrap,
    unwrap,
    wrapping,
    # Interface for reducibles
    __foldl__,
    getproperty(@__MODULE__, Symbol("@return_if_reduced")),
    getproperty(@__MODULE__, Symbol("@next")),
]

const _explicit_internal_list = Any[
    # Manually list some internal objects to avoid Documenter.jl's
    # "docstring potentially missing" warning:
    DefaultInit,
]

is_internal(@nospecialize(t)) =
    t ∈ _explicit_internal_list ||
    (parentmodule(t) === @__MODULE__) && t ∉ _unexported_public_api

is_transducer_type(t) = t isa Type && t <: Transducer && t !== Transducer
is_transducer_type(::typeof(Zip)) = true

struct TransducerLister
    m::Module
end

TransducerLister() = TransducerLister(@__MODULE__)

Transducer(tl::TransducerLister) = opcompose(
    Filter() do x
        t = getproperty(tl.m, x)
        is_transducer_type(t)
    end,
    Map(x -> Docs.Binding(tl.m, x)),
)

(tl::TransducerLister)() = eduction(Transducer(tl), names(tl.m))

header_code(m::Markdown.MD) =
    if !isempty(m.content)
        header_code(m.content[1])
    end
header_code(c::Markdown.Code) = c
header_code(::Any) = nothing

first_paragraph(m::Markdown.MD) =
    if length(m.content) == 1
        first_paragraph(m.content[1])
    elseif length(m.content) > 1
        first_paragraph(m.content[2])
    end
first_paragraph(p::Markdown.Paragraph) = p
first_paragraph(::Any) = nothing

hasdoc(m::Module, b::Docs.Binding) = haskey(Docs.meta(m), b)

function Base.show(io::IO, ::MIME"text/markdown", tl::TransducerLister)
    println(io, "| **Transducer** | **Summary** |")
    println(io, "|:-- |:-- |")
    foreach(tl()) do binding
        hasdoc(tl.m, binding) || return
        d = Docs.doc(binding)
        h = header_code(d)
        if h === nothing
            shown = binding.var
        else
            shown, = split(h.code, "\n", limit=2)
        end

        p = first_paragraph(d)
        if p === nothing
            gist = ""
        else
            gist, = split(sprint(show, "text/markdown", Markdown.MD(p)),
                          "\n", limit=2)
        end

        println(io,
                "| [`", shown, "`](@ref ", binding, ")",
                " | ", gist,
                " |")
    end
end

function Markdown.MD(tl::TransducerLister)
    io = IOBuffer()
    show(io, "text/markdown", tl)
    seek(io, 0)
    return Markdown.parse(io)
end

Base.show(io::IO, mime::MIME"text/plain", tl::TransducerLister) =
    _show(io, mime, tl)
Base.show(io::IO, mime::MIME"text/html", tl::TransducerLister) =
    _show(io, mime, tl)

_show(io, mime, tl) = show(io, mime, Markdown.MD(tl))


text:
function get_bdt(obj::PreloadedTimeScales)
    return jcall(obj, "getBDT", BDTScale, ())
end

function get_glonass(obj::PreloadedTimeScales)
    return jcall(obj, "getGLONASS", GLONASSScale, ())
end

function get_gps(obj::PreloadedTimeScales)
    return jcall(obj, "getGPS", GPSScale, ())
end

function get_gst(obj::PreloadedTimeScales)
    return jcall(obj, "getGST", GalileoScale, ())
end

function get_irnss(obj::PreloadedTimeScales)
    return jcall(obj, "getIRNSS", IRNSSScale, ())
end

function get_qzss(obj::PreloadedTimeScales)
    return jcall(obj, "getQZSS", QZSSScale, ())
end

function get_tai(obj::PreloadedTimeScales)
    return jcall(obj, "getTAI", TAIScale, ())
end

function get_tcb(obj::PreloadedTimeScales)
    return jcall(obj, "getTCB", TCBScale, ())
end

function get_tcg(obj::PreloadedTimeScales)
    return jcall(obj, "getTCG", TCGScale, ())
end

function get_tdb(obj::PreloadedTimeScales)
    return jcall(obj, "getTDB", TDBScale, ())
end

function get_tt(obj::PreloadedTimeScales)
    return jcall(obj, "getTT", TTScale, ())
end

function get_utc(obj::PreloadedTimeScales)
    return jcall(obj, "getUTC", UTCScale, ())
end


text:
using BinaryBuilder, Pkg

julia_version = v"1.6.0"

name = "RDKit"
version = v"2021.09.1pre"

sources = [
    GitSource("https://github.com/rdkit/rdkit.git", "d859a23c9e514e3266adeb7119ef337f87e7214b"),
]

script = raw"""
cd ${WORKSPACE}/srcdir/rdkit
mkdir build
cd build
cmake \
-DCMAKE_BUILD_TYPE=Release \
-DRDK_INSTALL_INTREE=OFF \
-DRDK_BUILD_INCHI_SUPPORT=ON \
-DRDK_BUILD_PYTHON_WRAPPERS=OFF \
-DRDK_BUILD_CFFI_LIB=ON \
-DRDK_BUILD_FREETYPE_SUPPORT=ON \
-DRDK_BUILD_CPP_TESTS=OFF \
-RDK_BUILD_SLN_SUPPORT=OFF \
-DCMAKE_INSTALL_PREFIX=${prefix} \
-DCMAKE_TOOLCHAIN_FILE=${CMAKE_TARGET_TOOLCHAIN} \
-DCMAKE_PREFIX_PATH=${prefix} \
..
make -j${nproc}
make install
"""

platforms = [
    Platform("x86_64", "linux"),
    Platform("i686", "linux"),
    Platform("aarch64", "linux"),
    Platform("x86_64", "macos"),
    # Platform("aarch64", "macos"),
]

platforms = expand_cxxstring_abis(platforms)

products = [
    LibraryProduct("librdkitcffi", :librdkitcffi),
]

dependencies = [
    Dependency("FreeType2_jll"),
    Dependency("boost_jll"),
    BuildDependency("Eigen_jll"),
    Dependency("Zlib_jll"),
]

build_tarballs(ARGS, name, version, sources, script, platforms, products, dependencies;
    preferred_gcc_version=v"5", julia_compat="^$(julia_version.major).$(julia_version.minor)")

max_stars_repo_name: waralex/Yggdrasil

max_issues_repo_name: waralex/Yggdrasil

max_forks_repo_name: waralex/Yggdrasil


text:
const KeyMap = (
    UP      = ["W", "UP"],
    DOWN    = ["S", "DOWN"],
    LEFT    = ["A", "LEFT"],
    RIGHT   = ["D", "RIGHT"],
)

const Settings = (
    framerate = 10,
    resolution = 20,
    colors = (
        bg = "#111",
        snake = "#eee",
        gameover = "red"
    )
)

const Dir = (
    UP    = [ 0, -1],
    DOWN  = [ 0,  1],
    LEFT  = [-1,  0],
    RIGHT = [ 1,  0]
)

max_stars_repo_name: jorge-brito/LynxGallery

max_issues_repo_name: jorge-brito/LynxGallery

max_forks_repo_name: jorge-brito/LynxGallery


text:
using Flux
using BSON
using LinearAlgebra
using Statistics
using Random
using Zygote: @adjoint
using Zygote: @nograd
using Zygote
using ProgressBars
using Distributions
using Flux.Losses: ctc_loss

Random.seed!(1)

const TRAINDIR = "train"

const EPOCHS = 100
const BATCH_SIZE = 1
const VAL_SIZE = 500

losses = []

forward = LSTM(39, 100)
backward = LSTM(39, 100)
output = Dense(200, 62)

const NOISE = Normal(0, 0.6)

function m(x)
  h0f = forward.(x)
  h0b = Flux.flip(backward, x)
  h0 = vcat.(h0f, h0b)
  o = output.(h0)
  return o
end

function loss(x, y)
  x = addNoise(x)
  Flux.reset!((forward, backward))
  yhat = m(x)
  yhat = reduce(hcat, yhat)
  l = ctc_loss(yhat, y)
  addToGlobalLoss(l)
  return l
end

@nograd function addNoise(x)
  return [xI .+ Float32.(rand(NOISE, 39)) for xI in x]
end

function readData(dataDir)
  fnames = readdir(dataDir)
  shuffle!(MersenneTwister(4), fnames)
  Xs = []
  Ys = []

  for fname in fnames
    BSON.@load joinpath(dataDir, fname) x y
    x = [Float32.(x[i,:]) for i in 1:size(x,1)]
    push!(Xs, x)
    push!(Ys, collapse(argmax.(eachcol(y))))
  end

  m = mean(reduce(vcat, Xs))
  st = std(reduce(vcat, Xs))
  for (i, x) in enumerate(Xs)
    Xs[i] = [(xI .- m) ./ st for xI in x]
  end

  return (Xs, Ys)
end

function lev(s, t)
    m = length(s)
    n = length(t)
    d = Array{Int}(zeros(m+1, n+1))

    for i=2:(m+1)
        @inbounds d[i, 1] = i-1
    end

    for j=2:(n+1)
        @inbounds d[1, j] = j-1
    end

    for j=2:(n+1)
        for i=2:(m+1)
            @inbounds if s[i-1] == t[j-1]
                substitutionCost = 0
            else
                substitutionCost = 1
            end
            @inbounds d[i, j] = min(d[i-1, j] + 1, # Deletion
                            d[i, j-1] + 1, # Insertion
                            d[i-1, j-1] + substitutionCost) # Substitution
        end
    end

    @inbounds return d[m+1, n+1]
end

function collapse(seq)
  if isempty(seq) return seq end
  s = [seq[1]]
  for ch in seq[2:end]
    if ch != s[end]
      push!(s, ch)
    end
  end
  filter!(x -> x != 62, s)
  return s
end

# collapses all repetitions into a single item and
# removes blanks; used because data set in its current
# form has one-hot encoded phones for every time step
# instead of just the phones contained in the sequence.
# With a proper re-extraction of the data, this would not
# be necessary.
function alt_collapse(seq)
  if isempty(seq) return seq end
  s = [seq[1]]
  for ch in seq[2:end]
    if ch != s[end]
      push!(s, ch)
    end
  end
  filter!(!=(62), s)
  return s
end

"""
    per(x, y)

Compute the phoneme error rate of the model for input `x` and target `y`. The phoneme error rate
is defined as the Levenshtein distance between the labeling produced by running `x` through
the model and the target labeling in `y`, all divided by the length of the target labeling
in `y`
"""
function per(x, y)
  Flux.reset!((forward, backward))
  yhat = m(x)
  yhat = reduce(hcat, yhat)
  yhat = mapslices(argmax, yhat, dims=1) |> vec |> alt_collapse
  # y = mapslices(argmax, y, dims=1) |> vec |> collapse
  return lev(yhat, y) / length(y)
end

function addToGlobalLoss(x)
	global losses
	push!(losses, x)
end

@adjoint function addToGlobalLoss(x)
	addToGlobalLoss(x)
	return nothing, () -> nothing
end

function main()
  println("Loading files")
  Xs, Ys = readData(TRAINDIR)
  data = collect(zip(Xs, Ys))

  Xs = [d[1] for d in data]
  Ys = [d[2] for d in data]

  println("Beginning training")

  data = zip(Xs, Ys) |> collect
  valData = data[1:VAL_SIZE]
  data = data[VAL_SIZE+1:end]

  opt = Momentum(1e-4)

  for i in 1:EPOCHS
    global losses
    losses = []
    println("Beginning epoch $i/$EPOCHS")
    Flux.train!(loss, Flux.params((forward, backward, output)), ProgressBar(data), opt)
    println("Calculating PER...")
    p = mean(map(x -> per(x...), valData))
    println("PER: $(p*100)")

    println("Mean loss: ", mean(losses))
  end
end

main()

max_stars_repo_name: maetshju/flux-ctc-grad

max_issues_repo_name: maetshju/flux-ctc-grad

max_forks_repo_name: maetshju/flux-ctc-grad


text:
export blasso, sfwsolver, momsossolver, solve!

mutable struct BLASSO{T, FT, DT}
    y::Array{T, 1}
    φ::FT
    λ::Real
    domain::DT

    μ::Union{AbstractMeasureLike, Nothing}
    objvalue::Union{Real, Nothing}
    dualsol::Union{Vector{Real}, Nothing}
    mommat::Union{MomentMatrix, Nothing}
end

function blasso(y::Array{T, 1}, φ::Array{PT, 1}, λ::Real,
                domain::AbstractSemialgebraicSet) where {T, PT<:AbstractPolynomialLike}
    BLASSO(y, φ, λ, domain, nothing, nothing, nothing, nothing)
end

function blasso(y::Array{T, 1}, φ::Function, λ::Real,
                domain::Array{BT, 2}) where {T, BT}
    BLASSO(y, φ, λ, domain, nothing, nothing, nothing, nothing)
end

struct Solver
    name::String
    options::Dict
end

function sfwsolver(niter::Integer, gridsize::Integer)
    Solver("sfw", Dict("niter" => niter, "gridsize" => gridsize))
end

function momsossolver(ϵ::Real, maxdeg::Union{Int, String}="defaut")
    Solver("momsos", Dict("ϵ" => ϵ, "maxdeg" => maxdeg))
end

function solve!(blasso::BLASSO, solver::Solver)
    if solver.name == "momsos"
        momsos!(blasso, solver.options["ϵ"], solver.options["maxdeg"])
    elseif solver.name == "sfw"
        sfw!(blasso, solver.options["niter"], solver.options["gridsize"])
    else
        throw(ArgumentError(solver.name,
                            "solver name must be either 'momsos' or 'sfw'"))
    end
end


text:
# Generic.
include("compiler/utils.jl")
include("compiler/loop_detection.jl")
include("compiler/reaching.jl")
include("compiler/address_blanket.jl")
include("compiler/absint/absint.jl")

# Kernel detection and dynamic addressing hints.
include("compiler/hints.jl")

# Trace types system
include("compiler/absint/trace_types.jl")

# Support error checker.
include("compiler/static/support_checker.jl")

# Partial evaluation/specialization
include("compiler/specializer/diffs.jl")
include("compiler/specializer/transforms.jl")
include("compiler/specializer/interface.jl")

# Automatic addressing converts stochastic functions into PPs
include("compiler/automatic_addressing/automatic_addressing_transform.jl")

# ------------ Documentation ------------ #


text:
using Test
using OrdinaryDiffEq, Calculus, ForwardDiff, FiniteDiff

function f(du,u,p,t)
  du[1] = -p[1]
  du[2] = p[2]
end

for x in 0:0.001:5
  called = false
  if x in [1.0, 2.0, 3.0, 4.0, 5.0]
    print("AD Ping $x")
  end
  function test_f(p)
    cb = ContinuousCallback((u,t,i) -> u[1], (integrator)->(called=true;integrator.p[2]=zero(integrator.p[2])))
    prob = ODEProblem(f,eltype(p).([1.0,0.0]),eltype(p).((0.0,1.0)),copy(p))
    integrator = init(prob,Tsit5(),abstol=1e-14,reltol=1e-14,callback=cb)
    step!(integrator)
    solve!(integrator).u[end]
  end
  p = [2.0, x]
  called = false
  findiff = Calculus.finite_difference_jacobian(test_f,p)
  @test called
  called = false
  fordiff = ForwardDiff.jacobian(test_f,p)
  @test called
  @test findiff ≈ fordiff
end

function f2(du,u,p,t)
  du[1] = -u[2]
  du[2] = p[2]
end

for x in 2.1:0.001:5
  called = false
  if x in [3.0, 4.0, 5.0]
    print("AD Ping $x")
  end
  function test_f2(p)
    cb = ContinuousCallback((u,t,i) -> u[1], (integrator)->(called=true;integrator.p[2]=zero(integrator.p[2])))
    prob = ODEProblem(f2,eltype(p).([1.0,0.0]),eltype(p).((0.0,1.0)),copy(p))
    integrator = init(prob,Tsit5(),abstol=1e-12,reltol=1e-12,callback=cb)
    step!(integrator)
    solve!(integrator).u[end]
  end
  p = [2.0, x]
  findiff = Calculus.finite_difference_jacobian(test_f2,p)
  @test called
  called = false
  fordiff = ForwardDiff.jacobian(test_f2,p)
  @test called
  @test findiff ≈ fordiff
end

#=
#x = 2.0 is an interesting case

x = 2.0

function test_f2(p)
  cb = ContinuousCallback((u,t,i) -> u[1], (integrator)->(@show(x,integrator.t);called=true;integrator.p[2]=zero(integrator.p[2])))
  prob = ODEProblem(f2,eltype(p).([1.0,0.0]),eltype(p).((0.0,1.0)),copy(p))
  integrator = init(prob,Tsit5(),abstol=1e-12,reltol=1e-12,callback=cb)
  step!(integrator)
  solve!(integrator).u[end]
end

p = [2.0, x]
findiff = Calculus.finite_difference_jacobian(test_f2,p)
@test called
called = false
fordiff = ForwardDiff.jacobian(test_f2,p)
@test called

# At that value, it shouldn't be called, but a small perturbation will make it called, so finite difference is wrong!
=#

for x in 1.0:0.001:2.5
  if x in [1.5, 2.0, 2.5]
    print("AD Ping $x")
  end
  function lotka_volterra(du,u,p,t)
    x, y = u
    α, β, δ, γ = p
    du[1] = dx = α*x - β*x*y
    du[2] = dy = -δ*y + γ*x*y
  end
  u0 = [1.0,1.0]
  tspan = (0.0,10.0)
  p = [x,1.0,3.0,1.0]
  prob = ODEProblem(lotka_volterra,u0,tspan,p)
  sol = solve(prob,Tsit5())

  called=false
  function test_lotka(p)
    cb = ContinuousCallback((u,t,i) -> u[1]-2.5, (integrator)->(called=true;integrator.p[4]=1.5))
    prob = ODEProblem(lotka_volterra,eltype(p).([1.0,1.0]),eltype(p).((0.0,10.0)),copy(p))
    integrator = init(prob,Tsit5(),abstol=1e-12,reltol=1e-12,callback=cb)
    step!(integrator)
    solve!(integrator).u[end]
  end

  findiff = Calculus.finite_difference_jacobian(test_lotka,p)
  @test called
  called = false
  fordiff = ForwardDiff.jacobian(test_lotka,p)
  @test called
  @test findiff ≈ fordiff
end

# Gradients and Hessians

function myobj(θ)
  f(u,p,t) = -θ[1]*u
  u0, _ = promote(10.0, θ[1])
  prob = ODEProblem(f, u0, (0.0, 1.0))
  sol = solve(prob, Tsit5())
  diff = sol.u - 10*exp.(-sol.t)
  return diff'diff
end

ForwardDiff.gradient(myobj, [1.0])
ForwardDiff.hessian(myobj, [1.0])

function myobj2(θ)
  f(du,u,p,t) = (du[1]=-θ[1]*u[1])
  u0, _ = promote(10.0, θ[1])
  prob = ODEProblem(f, [u0], (0.0, 1.0))
  sol = solve(prob, Tsit5())
  diff = sol[:,1] .- 10 .*exp.(-sol.t)
  return diff'diff
end

ForwardDiff.gradient(myobj2, [1.0])
ForwardDiff.hessian(myobj2, [1.0])

function myobj3(θ)
  f(u,p,t) = -θ[1]*u
  u0, _ = promote(10.0, θ[1])
  tspan_end, _ = promote(1.0, θ[1])
  prob = ODEProblem(f, u0, (0.0, tspan_end))
  sol = solve(prob, Tsit5())
  diff = sol.u - 10*exp.(-sol.t)
  return diff'diff
end

ForwardDiff.gradient(myobj3, [1.0])
ForwardDiff.hessian(myobj3, [1.0])

function myobj4(θ)
  f(du,u,p,t) = (du[1] = -θ[1]*u[1])
  u0, _ = promote(10.0, θ[1])
  tspan_end, _ = promote(1.0, θ[1])
  prob = ODEProblem(f, [u0], (0.0, tspan_end))
  sol = solve(prob, Tsit5())
  diff = sol[:,1] .- 10 .* exp.(-sol.t)
  return diff'diff
end

ForwardDiff.gradient(myobj4, [1.0])
ForwardDiff.hessian(myobj4, [1.0])

f1s = function (du,u,p,t)
    du[1] = p[2] * u[1]
    du[2] = p[3] * u[2]
    du[3] = p[4] * u[3]
    du[4] = p[5] * u[4]
    nothing
end
f2s = function (du,u,p,t)
    du[1] = p[1]*u[2]
    du[2] = p[1]*u[3]
    du[3] = p[1]*u[4]
    du[4] = p[1]*u[1]
    nothing
end
u0 = [3.4, 3.3, 3.2, 3.1]
params = [0.002, -0.005, -0.004, -0.003, -0.002]
tspan = (7.0, 84.0)
times = collect(minimum(tspan):0.5:maximum(tspan))
prob = SplitODEProblem(f1s,f2s, u0, tspan, params)
sol2 = solve(prob, KenCarp4(); dt=0.5, saveat=times)

function difffunc(p)
    tmp_prob = remake(prob,p=p)
    vec(solve(tmp_prob,KenCarp4(),saveat=times))
end
ForwardDiff.jacobian(difffunc,ones(5))

# https://github.com/SciML/OrdinaryDiffEq.jl/issues/1221

f_a  = function (du, u, p, t)
    du[1] = -p[1]*u[1] + exp(-t)
end

of_a = p -> begin
    u0  = [0.0]
    tspan = (0.0, 5.0)
    prob  = ODEProblem(f_a, u0, tspan, p)
    # sol = solve(prob, Tsit5())                      # works
    # sol = solve(prob, Rodas5(autodiff=false))       # works
    sol = solve(prob, Rodas5(autodiff=true),abstol=1e-14,reltol=1e-14)          # fails
    return sum(t -> abs2(t[1]), sol([1.0, 2.0, 3.0]))
end

@test !iszero(ForwardDiff.gradient(t -> of_a(t), [1.0]))
@test ForwardDiff.gradient(t -> of_a(t), [1.0]) ≈ FiniteDiff.finite_difference_gradient(t -> of_a(t), [1.0]) rtol=1e-5

SOLVERS_FOR_AD = (
    (BS3         , 1e-12),
    (Tsit5       , 1e-12),
    (KenCarp4    , 1e-11),
    (KenCarp47   , 1e-11),
    (KenCarp5    , 1e-11),
    (KenCarp58   , 1e-12),
    (TRBDF2      , 1e-07),
    (Rodas4      , 1e-12),
    (Rodas5      , 1e-12),
    (Rosenbrock23, 1e-10),
    (Rosenbrock32, 1e-11),
    (Vern6       , 1e-11),
    (Vern7       , 1e-11),
    (RadauIIA3   , 1e-04),
    (RadauIIA5   , 1e-12),
)

@testset "$alg can handle ForwardDiff.Dual in u0 with rtol=$rtol when iip=$iip" for
    (alg, rtol) in SOLVERS_FOR_AD,
    iip in (true, false)

    if iip
      f = (du, u, p, t) -> du .= -0.5*u
    else
      f = (u, p, t) -> -0.5*u
    end

    g = u0 -> begin
        tspan = (0.0, 1.0)
        prob = ODEProblem(
            f,
            u0,
            tspan
        )
        solve(prob, alg(), abstol=1e-14, reltol=1e-14)(last(tspan))[1]
    end
    @test ForwardDiff.gradient(g, [10.0])[1] ≈ exp(-0.5) rtol=rtol
end

@testset "$alg can handle ForwardDiff.Dual in t0 with rtol=$rtol when iip=$iip" for
    (alg, rtol) in SOLVERS_FOR_AD,
    iip in (true, false)

    if iip
      f = (du, u, p, t) -> du .= -0.5*u
    else
      f = (u, p, t) -> -0.5*u
    end

    _u0 = 10.0
    g = t0 -> begin
        tspan = (t0, 1.0)
        u0 = typeof(t0)[_u0]
        prob = ODEProblem(
            f,
            u0,
            tspan
        )
        solve(prob, alg(), abstol=1e-14, reltol=1e-14)(last(tspan))[1]
    end
    @test ForwardDiff.derivative(g, 0.0) ≈ _u0/2*exp(-0.5) rtol=rtol
end