text:
# This file is a part of Julia. License is MIT: https://julialang.org/license

module Types

using UUIDs
using Random
using Dates
import LibGit2
import REPL
using REPL.TerminalMenus

using ..TOML
import ..Pkg, ..UPDATED_REGISTRY_THIS_SESSION
import Pkg: GitTools, depots, depots1, logdir

import Base: SHA1
using SHA

export UUID, pkgID, SHA1, VersionRange, VersionSpec, empty_versionspec,
    Requires, Fixed, merge_requires!, satisfies, ResolverError,
    PackageSpec, EnvCache, Context, Context!, get_deps,
    PkgError, pkgerror, has_name, has_uuid, write_env, parse_toml, find_registered!,
    project_resolve!, project_deps_resolve!, manifest_resolve!, registry_resolve!, stdlib_resolve!, handle_repos_develop!, handle_repos_add!, ensure_resolved,
    manifest_info, registered_uuids, registered_paths, registered_uuid, registered_name,
    read_project, read_package, read_manifest, pathrepr, registries,
    PackageMode, PKGMODE_MANIFEST, PKGMODE_PROJECT, PKGMODE_COMBINED,
    UpgradeLevel, UPLEVEL_FIXED, UPLEVEL_PATCH, UPLEVEL_MINOR, UPLEVEL_MAJOR,
    PackageSpecialAction, PKGSPEC_NOTHING, PKGSPEC_PINNED, PKGSPEC_FREED, PKGSPEC_DEVELOPED, PKGSPEC_TESTED, PKGSPEC_REPO_ADDED,
    printpkgstyle,
    projectfile_path,
    RegistrySpec

include("versions.jl")

## ordering of UUIDs ##

Base.isless(a::UUID, b::UUID) = a.value < b.value

## Computing UUID5 values from (namespace, key) pairs ##
function uuid5(namespace::UUID, key::String)
    data = [reinterpret(UInt8, [namespace.value]); codeunits(key)]
    u = reinterpret(UInt128, sha1(data)[1:16])[1]
    u &= 0xffffffffffff0fff3fffffffffffffff
    u |= 0x00000000000050008000000000000000
    return UUID(u)
end
uuid5(namespace::UUID, key::AbstractString) = uuid5(namespace, String(key))

const uuid_dns = UUID(0x6ba7b810_9dad_11d1_80b4_00c04fd430c8)
const uuid_julia_project = uuid5(uuid_dns, "julialang.org")
const uuid_package = uuid5(uuid_julia_project, "package")
const uuid_registry = uuid5(uuid_julia_project, "registry")
const uuid_julia = uuid5(uuid_package, "julia")

## user-friendly representation of package IDs ##
function pkgID(p::UUID, uuid_to_name::Dict{UUID,String})
    name = get(uuid_to_name, p, "")
    isempty(name) && (name = "(unknown)")
    uuid_short = string(p)[1:8]
    return "$name [$uuid_short]"
end

####################
# Requires / Fixed #
####################
const Requires = Dict{UUID,VersionSpec}

function merge_requires!(A::Requires, B::Requires)
    for (pkg, vers) in B
        A[pkg] = haskey(A, pkg) ? intersect(A[pkg], vers) : vers
    end
    return A
end

satisfies(pkg::UUID, ver::VersionNumber, reqs::Requires) =
    !haskey(reqs, pkg) || in(ver, reqs[pkg])

struct Fixed
    version::VersionNumber
    requires::Requires
end
Fixed(v::VersionNumber) = Fixed(v, Requires())

Base.:(==)(a::Fixed, b::Fixed) = a.version == b.version && a.requires == b.requires
Base.hash(f::Fixed, h::UInt) = hash((f.version, f.requires), h + (0x68628b809fd417ca % UInt))

Base.show(io::IO, f::Fixed) = isempty(f.requires) ?
    print(io, "Fixed(", repr(f.version), ")") :
    print(io, "Fixed(", repr(f.version), ",", f.requires, ")")

struct ResolverError <: Exception
    msg::AbstractString
    ex::Union{Exception,Nothing}
end
ResolverError(msg::AbstractString) = ResolverError(msg, nothing)

function Base.showerror(io::IO, pkgerr::ResolverError)
    print(io, pkgerr.msg)
    if pkgerr.ex !== nothing
        pkgex = pkgerr.ex
        if isa(pkgex, CompositeException)
            for cex in pkgex
                print(io, "\n=> ")
                showerror(io, cex)
            end
        else
            print(io, "\n")
            showerror(io, pkgex)
        end
    end
end

#################
# Pkg Error #
#################
struct PkgError <: Exception
    msg::String
end
pkgerror(msg::String...) = throw(PkgError(join(msg)))
Base.show(io::IO, err::PkgError) = print(io, err.msg)

###############
# PackageSpec #
###############
@enum(UpgradeLevel, UPLEVEL_FIXED, UPLEVEL_PATCH, UPLEVEL_MINOR, UPLEVEL_MAJOR)
@enum(PackageMode, PKGMODE_PROJECT, PKGMODE_MANIFEST, PKGMODE_COMBINED)
@enum(PackageSpecialAction, PKGSPEC_NOTHING, PKGSPEC_PINNED, PKGSPEC_FREED,
                            PKGSPEC_DEVELOPED, PKGSPEC_TESTED, PKGSPEC_REPO_ADDED)

const VersionTypes = Union{VersionNumber,VersionSpec,UpgradeLevel}

# The url field can also be a local path, rename?
mutable struct GitRepo
    url::String
    rev::String
    git_tree_sha1::Union{Nothing,SHA1}
end

GitRepo(url::String, revspec) = GitRepo(url, revspec, nothing)
GitRepo(url::String) = GitRepo(url, "", nothing)
GitRepo(;url::Union{String, Nothing}=nothing, rev::Union{String, Nothing} =nothing) =
    GitRepo(url == nothing ? "" : url, rev == nothing ? "" : rev, nothing)
Base.:(==)(repo1::GitRepo, repo2::GitRepo) = (repo1.url == repo2.url && repo1.rev == repo2.rev && repo1.git_tree_sha1 == repo2.git_tree_sha1)

mutable struct PackageSpec
    name::String
    uuid::UUID
    version::VersionTypes
    mode::PackageMode
    path::Union{Nothing,String}
    special_action::PackageSpecialAction # If the package is currently being pinned, freed etc
    repo::Union{Nothing,GitRepo}
end
PackageSpec(name::AbstractString, uuid::UUID, version::VersionTypes,
            mode::PackageMode=PKGMODE_PROJECT, path=nothing, special_action=PKGSPEC_NOTHING,
            repo=nothing) = PackageSpec(String(name), uuid, version, mode, path, special_action, repo)
PackageSpec(name::AbstractString, uuid::UUID) =
    PackageSpec(name, uuid, VersionSpec())
PackageSpec(name::AbstractString, version::VersionTypes=VersionSpec()) =
    PackageSpec(name, UUID(zero(UInt128)), version)
PackageSpec(uuid::UUID, version::VersionTypes=VersionSpec()) =
    PackageSpec("", uuid, version)
function PackageSpec(repo::GitRepo)
    pkg = PackageSpec()
    pkg.repo = repo
    return pkg
end

# kwarg constructor
function PackageSpec(;name::AbstractString="", uuid::Union{String, UUID}=UUID(0),
                     version::Union{VersionNumber, String, VersionSpec} = VersionSpec(),
                     url = nothing, rev = nothing, path=nothing, mode::PackageMode = PKGMODE_PROJECT)
    if url !== nothing || path !== nothing || rev !== nothing
        if path !== nothing || url !== nothing
            path !== nothing && url !== nothing && pkgerror("cannot specify both path and url")
            url = url == nothing ? path : url
        end
        repo = GitRepo(url=url, rev=rev)
    else
        repo = nothing
    end

    version = VersionSpec(version)
    uuid isa String && (uuid = UUID(uuid))
    PackageSpec(name, uuid, version, mode, nothing, PKGSPEC_NOTHING, repo)
end

has_name(pkg::PackageSpec) = !isempty(pkg.name)
has_uuid(pkg::PackageSpec) = pkg.uuid != UUID(zero(UInt128))

function Base.show(io::IO, pkg::PackageSpec)
    vstr = repr(pkg.version)
    f = ["name" => pkg.name, "uuid" => has_uuid(pkg) ? pkg.uuid : "", "v" => (vstr == "VersionSpec(\"*\")" ? "" : vstr)]
    if pkg.repo !== nothing
        if !isempty(pkg.repo.url)
            push!(f, "url/path" => string("\"", pkg.repo.url, "\""))
        end
        if !isempty(pkg.repo.rev)
            push!(f, "rev" => pkg.repo.rev)
        end
    end
    print(io, "PackageSpec(")
    first = true
    for (field, value) in f
        value == "" && continue
        first || print(io, ", ")
        print(io, field, "=", value)
        first = false
    end
    print(io, ")")
end

############
# EnvCache #
############

function parse_toml(path::String...; fakeit::Bool=false)
    p = joinpath(path...)
    !fakeit || isfile(p) ? TOML.parsefile(p) : Dict{String,Any}()
end

let trynames(names) = begin
    return root_path::AbstractString -> begin
        for x in names
            maybe_file = joinpath(root_path, x)
            if isfile(maybe_file)
                return maybe_file
            end
        end
    end
end # trynames
    global projectfile_path = trynames(Base.project_names)
    global manifestfile_path = trynames(Base.manifest_names)
end # let

function find_project_file(env::Union{Nothing,String}=nothing)
    project_file = nothing
    if env isa Nothing
        project_file = Base.active_project()
        project_file == nothing && pkgerror("no active project")
    elseif startswith(env, '@')
        project_file = Base.load_path_expand(env)
        project_file === nothing && pkgerror("package environment does not exist: $env")
    elseif env isa String
        if isdir(env)
            isempty(readdir(env)) || pkgerror("environment is a package directory: $env")
            project_file = joinpath(env, Base.project_names[end])
        else
            project_file = endswith(env, ".toml") ? abspath(env) :
                abspath(env, Base.project_names[end])
        end
    end
    @assert project_file isa String &&
        (isfile(project_file) || !ispath(project_file) ||
         isdir(project_file) && isempty(readdir(project_file)))
    return safe_realpath(project_file)
end

mutable struct EnvCache
    # environment info:
    env::Union{Nothing,String}
    git::Union{Nothing,String}

    # paths for files:
    project_file::String
    manifest_file::String

    # name / uuid of the project
    pkg::Union{PackageSpec, Nothing}

    # cache of metadata:
    project::Dict
    manifest::Dict

    # registered package info:
    uuids::Dict{String,Vector{UUID}}
    paths::Dict{UUID,Vector{String}}
    names::Dict{UUID,Vector{String}}
end

function EnvCache(env::Union{Nothing,String}=nothing)
    project_file = find_project_file(env)
    project_dir = dirname(project_file)
    git = ispath(joinpath(project_dir, ".git")) ? project_dir : nothing

    project = read_project(project_file)
    if any(haskey.((project,), ["name", "uuid", "version"]))
        project_package = PackageSpec(
            get(project, "name", ""),
            UUID(get(project, "uuid", 0)),
            VersionNumber(get(project, "version", "0.0")),
        )
    else
        project_package = nothing
    end
    # determine manifest_file name
    dir = abspath(dirname(project_file))
    manifest_file = haskey(project, "manifest") ?
        abspath(project["manifest"]) :
        manifestfile_path(dir)
    # use default name if still not determined
    (manifest_file === nothing) && (manifest_file = joinpath(dir, "Manifest.toml"))
    write_env_usage(manifest_file)
    manifest = read_manifest(manifest_file)
    uuids = Dict{String,Vector{UUID}}()
    paths = Dict{UUID,Vector{String}}()
    names = Dict{UUID,Vector{String}}()
    return EnvCache(env,
        git,
        project_file,
        manifest_file,
        project_package,
        project,
        manifest,
        uuids,
        paths,
        names,)
end

collides_with_project(env::EnvCache, pkg::PackageSpec) =
    is_project_name(env, pkg.name) || is_project_uuid(env, pkg.uuid)
is_project(env::EnvCache, pkg::PackageSpec) = is_project_uuid(env, pkg.uuid)
is_project_name(env::EnvCache, name::String) =
    env.pkg !== nothing && env.pkg.name == name
is_project_uuid(env::EnvCache, uuid::UUID) =
    env.pkg !== nothing && env.pkg.uuid == uuid

###########
# Context #
###########
stdlib_dir() = normpath(joinpath(Sys.BINDIR, "..", "share", "julia", "stdlib", "v$(VERSION.major).$(VERSION.minor)"))
stdlib_path(stdlib::String) = joinpath(stdlib_dir(), stdlib)
function gather_stdlib_uuids()
    stdlibs = Dict{UUID,String}()
    for stdlib in readdir(stdlib_dir())
        projfile = projectfile_path(stdlib_path(stdlib))
        if nothing !== projfile
            proj = TOML.parsefile(projfile)
            if haskey(proj, "uuid")
                stdlibs[UUID(proj["uuid"])] = stdlib
            end
        end
    end
    return stdlibs
end

# ENV variables to set some of these defaults?
Base.@kwdef mutable struct Context
    env::EnvCache = EnvCache()
    preview::Bool = false
    use_libgit2_for_all_downloads::Bool = false
    use_only_tarballs_for_downloads::Bool = false
    # NOTE: The JULIA_PKG_CONCURRENCY environment variable is likely to be removed in
    # the future. It currently stands as an unofficial workaround for issue #795.
    num_concurrent_downloads::Int = haskey(ENV, "JULIA_PKG_CONCURRENCY") ? parse(Int, ENV["JULIA_PKG_CONCURRENCY"]) : 8
    graph_verbose::Bool = false
    stdlibs::Dict{UUID,String} = gather_stdlib_uuids()
    # Remove next field when support for Pkg2 CI scripts is removed
    currently_running_target::Bool = false
    old_pkg2_clone_name::String = ""
end

function Context!(kw_context::Vector{Pair{Symbol,Any}})::Context
    ctx = Context()
    for (k, v) in kw_context
        setfield!(ctx, k, v)
    end
    return ctx
end

function Context!(ctx::Context; kwargs...)
    for (k, v) in kwargs
        setfield!(ctx, k, v)
    end
end

# target === nothing : main dependencies
# target === "*"     : main + all extras
# target === "name"  : named target deps

function deps_names(project::Dict, target::Union{Nothing,String}=nothing)::Vector{String}
    deps = sort!(collect(keys(project["deps"])))
    target == "*" && return !haskey(project, "extras") ? deps :
        sort!(union!(deps, collect(keys(project["extras"]))))
    haskey(project, "targets") || return deps
    targets = project["targets"]
    haskey(targets, target) || return deps
    return sort!(union!(deps, targets[target]))
end

function get_deps(project::Dict, target::Union{Nothing,String}=nothing)
    names = deps_names(project, target)
    deps = filter(((dep, _),) -> dep in names, project["deps"])
    extras = get(project, "extras", Dict{String,Any}())
    for name in names
        haskey(deps, name) && continue
        haskey(extras, name) ||
            pkgerror("target `$target` has unlisted dependency `$name`")
        deps[name] = extras[name]
    end
    return deps
end
get_deps(env::EnvCache, target::Union{Nothing,String}=nothing) =
    get_deps(env.project, target)
get_deps(ctx::Context, target::Union{Nothing,String}=nothing) =
    get_deps(ctx.env, target)

function project_compatibility(ctx::Context, name::String)
    v = VersionSpec()
    project = ctx.env.project
    compat = get(project, "compat", Dict())
    if haskey(compat, name)
        v = VersionSpec(semver_spec(compat[name]))
    end
    return v
end

function write_env_usage(manifest_file::AbstractString)
    !ispath(logdir()) && mkpath(logdir())
    usage_file = joinpath(logdir(), "manifest_usage.toml")
    touch(usage_file)
    !isfile(manifest_file) && return
    # Do not rewrite as do syntax (no longer precompilable)
    io = open(usage_file, "a")
    println(io, "[[\"", escape_string(manifest_file), "\"]]")
    print(io, "time = ", now()); println(io, 'Z')
    close(io)
end

function read_project(io::IO)
    project = TOML.parse(io)
    if !haskey(project, "deps")
        project["deps"] = Dict{String,Any}()
    end
    return project
end
function read_project(file::String)
    isfile(file) ? open(read_project, file) : read_project(devnull)
end

_throw_package_err(x, f) = pkgerror("expected a `$x` entry in project file at $(abspath(f))")
function read_package(f::String)
    project = read_project(f)
    haskey(project, "name") || _throw_package_err("name", f)
    haskey(project, "uuid") || _throw_package_err("uuid", f)
    name = project["name"]
    entry = joinpath(dirname(f), "src", "$name.jl")
    if !isfile(entry)
        pkgerror("expected the file `src/$name.jl` to exist for package $name at $(dirname(f))")
    end
    return project
end

function read_manifest(io::IO)
    manifest = TOML.parse(io)
    for (name, infos) in manifest, info in infos
        haskey(info, "deps") || continue
        info["deps"] isa AbstractVector || continue
        for dep in info["deps"]
            length(manifest[dep]) == 1 ||
                error("ambiguious dependency for $name: $dep")
        end
        new_dict = Dict()
        for d in info["deps"]
            new_dict[d] = manifest[d][1]["uuid"]
        end
        info["deps"] = new_dict
    end
    return manifest
end
function read_manifest(file::String)
    try isfile(file) ? open(read_manifest, file) : read_manifest(devnull)
    catch err
        err isa ErrorException && startswith(err.msg, "ambiguious dependency") || rethrow()
        err.msg *= "In manifest file: $file"
        rethrow(err)
    end
end

const refspecs = ["+refs/*:refs/remotes/cache/*"]
const reg_pkg = r"(?:^|[\/\\])(\w+?)(?:\.jl)?(?:\.git)?(?:[\/\\])?$"

# Windows sometimes throw on `isdir`...
function isdir_windows_workaround(path::String)
    try isdir(path)
    catch e
        false
    end
end

# try to call realpath on as much as possible
function safe_realpath(path)
    ispath(path) && return realpath(path)
    a, b = splitdir(path)
    return joinpath(safe_realpath(a), b)
end
function relative_project_path(ctx::Context, path::String)
    # compute path relative the project
    # realpath needed to expand symlinks before taking the relative path
    return relpath(safe_realpath(abspath(path)),
                   safe_realpath(dirname(ctx.env.project_file)))
end

casesensitive_isdir(dir::String) = isdir_windows_workaround(dir) && basename(dir) in readdir(joinpath(dir, ".."))

function handle_repos_develop!(ctx::Context, pkgs::AbstractVector{PackageSpec}; shared::Bool)
    Base.shred!(LibGit2.CachedCredentials()) do creds
        env = ctx.env
        new_uuids = UUID[]
        for pkg in pkgs
            pkg.repo == nothing && continue
            pkg.special_action = PKGSPEC_DEVELOPED
            isempty(pkg.repo.url) && set_repo_for_pkg!(env, pkg)

            if isdir_windows_workaround(pkg.repo.url)
                # Developing a local package, just point `pkg.path` to it
                if isabspath(pkg.repo.url)
                    # absolute paths should stay absolute
                    pkg.path = pkg.repo.url
                else
                    # Relative paths are given relative pwd() so we
                    # translate that to be relative the project instead.
                    pkg.path = relative_project_path(ctx, pkg.repo.url)
                end
                folder_already_downloaded = true
                project_path = pkg.repo.url
                parse_package!(ctx, pkg, project_path)
            else
                # Only update the registry in case of developing a non-local package
                UPDATED_REGISTRY_THIS_SESSION[] || update_registries(ctx)
                # We save the repo in case another environement wants to
                # develop from the same repo, this avoids having to reclone it
                # from scratch.
                clone_path = joinpath(depots1(), "clones")
                mkpath(clone_path)
                repo_path = joinpath(clone_path, string(hash(pkg.repo.url), "_full"))
                repo = nothing
                try
                    repo, just_cloned = ispath(repo_path) ? (LibGit2.GitRepo(repo_path), false) : begin
                        r = GitTools.clone(pkg.repo.url, repo_path)
                        GitTools.fetch(r, pkg.repo.url; refspecs=refspecs, credentials=creds)
                        r, true
                    end
                    if !just_cloned
                        GitTools.fetch(repo, pkg.repo.url; refspecs=refspecs, credentials=creds)
                    end
                finally
                    repo isa LibGit2.GitRepo && LibGit2.close(repo)
                end

                # Copy the repo to a temporary place and check out the rev
                project_path = mktempdir()
                cp(repo_path, project_path; force=true)
                LibGit2.with(LibGit2.GitRepo(project_path)) do repo
                    if LibGit2.isattached(repo)
                        rev = LibGit2.branch(repo)
                    else
                        rev = string(LibGit2.GitHash(LibGit2.head(repo)))
                    end
                    gitobject, isbranch = get_object_branch(repo, rev, creds)
                    try
                        LibGit2.transact(repo) do r
                            if isbranch
                                LibGit2.branch!(r, rev, track=LibGit2.Consts.REMOTE_ORIGIN)
                            else
                                LibGit2.checkout!(r, string(LibGit2.GitHash(gitobject)))
                            end
                        end
                    finally
                        close(gitobject)
                    end
                end

                parse_package!(ctx, pkg, project_path)
                devdir = shared ? Pkg.devdir() : joinpath(dirname(ctx.env.project_file), "dev")
                dev_pkg_path = joinpath(devdir, pkg.name)
                if isdir(dev_pkg_path)
                    if !isfile(joinpath(dev_pkg_path, "src", pkg.name * ".jl"))
                        pkgerror("Path `$(dev_pkg_path)` exists but it does not contain `src/$(pkg.name).jl")
                    else
                        @info "Path `$(dev_pkg_path)` exists and looks like the correct package, using existing path instead of cloning"
                    end
                else
                    mkpath(dev_pkg_path)
                    mv(project_path, dev_pkg_path; force=true)
                    push!(new_uuids, pkg.uuid)
                end
                # Save the path as relative if it is a --local dev,
                # otherwise put in the absolute path.
                pkg.path = shared ? dev_pkg_path : relative_project_path(ctx, dev_pkg_path)
                Base.rm(project_path; force = true, recursive = true)
            end
            @assert pkg.path != nothing
            @assert has_uuid(pkg)
        end
        return new_uuids
    end
end

function handle_repos_add!(ctx::Context, pkgs::AbstractVector{PackageSpec};
                           upgrade_or_add::Bool=true, credentials=nothing)
    # Always update the registry when adding
    UPDATED_REGISTRY_THIS_SESSION[] || update_registries(ctx)
    creds = credentials !== nothing ? credentials : LibGit2.CachedCredentials()
    try
        env = ctx.env
        new_uuids = UUID[]
        for pkg in pkgs
            pkg.repo == nothing && continue
            pkg.special_action = PKGSPEC_REPO_ADDED
            isempty(pkg.repo.url) && set_repo_for_pkg!(env, pkg)
            clones_dir = joinpath(depots1(), "clones")
            mkpath(clones_dir)
            repo_path = joinpath(clones_dir, string(hash(pkg.repo.url)))
            repo = nothing
            do_nothing_more = false
            project_path = nothing
            folder_already_downloaded = false
            try
                repo = if ispath(repo_path)
                    LibGit2.GitRepo(repo_path)
                else
                    GitTools.clone(pkg.repo.url, repo_path, isbare=true, credentials=creds)
                end
                info = manifest_info(env, pkg.uuid)
                pinned = (info != nothing && get(info, "pinned", false))
                upgrading = upgrade_or_add && !pinned
                if upgrading
                    GitTools.fetch(repo; refspecs=refspecs, credentials=creds)
                    rev = pkg.repo.rev
                    # see if we can get rev as a branch
                    if isempty(rev)
                        if LibGit2.isattached(repo)
                            rev = LibGit2.branch(repo)
                        else
                            rev = string(LibGit2.GitHash(LibGit2.head(repo)))
                        end
                    end
                else
                    # Not upgrading so the rev should be the current git-tree-sha
                    rev = info["git-tree-sha1"]
                    pkg.version = VersionNumber(info["version"])
                end

                gitobject, isbranch = get_object_branch(repo, rev, creds)
                # If the user gave a shortened commit SHA, might as well update it to the full one
                try
                    if upgrading
                        pkg.repo.rev = isbranch ? rev : string(LibGit2.GitHash(gitobject))
                    end
                    LibGit2.with(LibGit2.peel(LibGit2.GitTree, gitobject)) do git_tree
                        @assert git_tree isa LibGit2.GitTree
                        pkg.repo.git_tree_sha1 = SHA1(string(LibGit2.GitHash(git_tree)))
                            version_path = nothing
                            folder_already_downloaded = false
                        if has_uuid(pkg) && has_name(pkg)
                            version_path = Pkg.Operations.find_installed(pkg.name, pkg.uuid, pkg.repo.git_tree_sha1)
                            isdir(version_path) && (folder_already_downloaded = true)
                            info = manifest_info(env, pkg.uuid)
                            if info != nothing && get(info, "git-tree-sha1", "") == string(pkg.repo.git_tree_sha1) && folder_already_downloaded
                                # Same tree sha and this version already downloaded, nothing left to do
                                do_nothing_more = true
                            end
                        end
                        if folder_already_downloaded
                            project_path = version_path
                        else
                            project_path = mktempdir()
                            opts = LibGit2.CheckoutOptions(checkout_strategy=LibGit2.Consts.CHECKOUT_FORCE,
                                target_directory=Base.unsafe_convert(Cstring, project_path))
                            LibGit2.checkout_tree(repo, git_tree, options=opts)
                        end
                    end
                finally
                    close(gitobject)
                end
            finally
                repo isa LibGit2.GitRepo && close(repo)
            end
            do_nothing_more && continue
            parse_package!(ctx, pkg, project_path)
            if !folder_already_downloaded
                version_path = Pkg.Operations.find_installed(pkg.name, pkg.uuid, pkg.repo.git_tree_sha1)
                mkpath(version_path)
                mv(project_path, version_path; force=true)
                push!(new_uuids, pkg.uuid)
            end
            Base.rm(project_path; force = true, recursive = true)
            @assert has_uuid(pkg)
        end
        return new_uuids
    finally
        creds !== credentials && Base.shred!(creds)
    end
end

function parse_package!(ctx, pkg, project_path)
    env = ctx.env
    project_file = projectfile_path(project_path)
    if project_file !== nothing
        project_data = read_package(project_file)
        pkg.uuid = UUID(project_data["uuid"])
        pkg.name = project_data["name"]
    else
        if !isempty(ctx.old_pkg2_clone_name) # remove when legacy CI script support is removed
            pkg.name = ctx.old_pkg2_clone_name
        else
            # This is an old style package, if not set, get the name from src/PackageName
            if !has_name(pkg)
                if isdir_windows_workaround(pkg.repo.url)
                    m = match(reg_pkg, abspath(pkg.repo.url))
                else
                    m = match(reg_pkg, pkg.repo.url)
                end
                m === nothing && pkgerror("cannot determine package name from URL or path: $(pkg.repo.url), provide a name argument to `PackageSpec`")
                pkg.name = m.captures[1]
            end
        end
        reg_uuids = registered_uuids(env, pkg.name)
        is_registered = !isempty(reg_uuids)
        if !is_registered
            # This is an unregistered old style package, give it a UUID and a version
            if !has_uuid(pkg)
                uuid_unreg_pkg = UUID(0xa9a2672e746f11e833ef119c5b888869)
                pkg.uuid = uuid5(uuid_unreg_pkg, pkg.name)
                @info "Assigning UUID $(pkg.uuid) to $(pkg.name)"
            end
        else
            @assert length(reg_uuids) == 1
            pkg.uuid = reg_uuids[1]
        end
    end
end

function set_repo_for_pkg!(env, pkg)
    if !has_uuid(pkg)
        registry_resolve!(env, [pkg])
        ensure_resolved(env, [pkg]; registry=true)
    end
    # TODO: look into [1]
    _, pkg.repo.url = Types.registered_info(env, pkg.uuid, "repo")[1]
end

function get_object_branch(repo, rev, creds)
    gitobject = nothing
    isbranch = false
    try
        gitobject = LibGit2.GitObject(repo, "remotes/cache/heads/" * rev)
        isbranch = true
    catch err
        err isa LibGit2.GitError && err.code == LibGit2.Error.ENOTFOUND || rethrow()
    end
    if gitobject == nothing
        try
            gitobject = LibGit2.GitObject(repo, rev)
        catch err
            err isa LibGit2.GitError && err.code == LibGit2.Error.ENOTFOUND || rethrow()
            GitTools.fetch(repo; refspecs=refspecs, credentials=creds)
            try
                gitobject = LibGit2.GitObject(repo, rev)
            catch err
                err isa LibGit2.GitError && err.code == LibGit2.Error.ENOTFOUND || rethrow()
                pkgerror("git object $(rev) could not be found")
            end
        end
    end
    return gitobject, isbranch
end

########################################
# Resolving packages from name or uuid #
########################################

function project_resolve!(env::EnvCache, pkgs::AbstractVector{PackageSpec})
    for pkg in pkgs
        if has_uuid(pkg) && !has_name(pkg) && Types.is_project_uuid(env, pkg.uuid)
            pkg.name = env.pkg.name
        end
        if has_name(pkg) && !has_uuid(pkg) && Types.is_project_name(env, pkg.name)
            pkg.uuid = env.pkg.uuid
        end
    end
end

# Disambiguate name/uuid package specifications using project info.
function project_deps_resolve!(env::EnvCache, pkgs::AbstractVector{PackageSpec})
    uuids = env.project["deps"]
    names = Dict(uuid => name for (name, uuid) in uuids)
    length(uuids) < length(names) && # TODO: handle this somehow?
        pkgerror("duplicate UUID found in project file's [deps] section")
    for pkg in pkgs
        pkg.mode == PKGMODE_PROJECT || continue
        if has_name(pkg) && !has_uuid(pkg) && pkg.name in keys(uuids)
            pkg.uuid = UUID(uuids[pkg.name])
        end
        if has_uuid(pkg) && !has_name(pkg) && pkg.uuid in keys(names)
            pkg.name = names[pkg.uuid]
        end
    end
    return pkgs
end

# Disambiguate name/uuid package specifications using manifest info.
function manifest_resolve!(env::EnvCache, pkgs::AbstractVector{PackageSpec})
    uuids = Dict{String,Vector{String}}()
    names = Dict{String,String}()
    for (name, infos) in env.manifest, info in infos
        haskey(info, "uuid") || continue
        uuid = info["uuid"]
        push!(get!(uuids, name, String[]), uuid)
        names[uuid] = name # can be duplicate but doesn't matter
    end
    for pkg in pkgs
        pkg.mode == PKGMODE_MANIFEST || continue
        if has_name(pkg) && !has_uuid(pkg) && pkg.name in keys(uuids)
            length(uuids[pkg.name]) == 1 && (pkg.uuid = UUID(uuids[pkg.name][1]))
        end
        if has_uuid(pkg) && !has_name(pkg) && pkg.uuid in keys(names)
            pkg.name = names[pkg.uuid]
        end
    end
    return pkgs
end

# Disambiguate name/uuid package specifications using registry info.
function registry_resolve!(env::EnvCache, pkgs::AbstractVector{PackageSpec})
    # if there are no half-specified packages, return early
    any(pkg -> has_name(pkg) ⊻ has_uuid(pkg), pkgs) || return
    # collect all names and uuids since we're looking anyway
    names = [pkg.name for pkg in pkgs if has_name(pkg)]
    uuids = [pkg.uuid for pkg in pkgs if has_uuid(pkg)]
    find_registered!(env, names, uuids)
    for pkg in pkgs
        @assert has_name(pkg) || has_uuid(pkg)
        if has_name(pkg) && !has_uuid(pkg)
            pkg.uuid = registered_uuid(env, pkg.name)
        end
        if has_uuid(pkg) && !has_name(pkg)
            pkg.name = registered_name(env, pkg.uuid)
        end
    end
    return pkgs
end

function stdlib_resolve!(ctx::Context, pkgs::AbstractVector{PackageSpec})
    for pkg in pkgs
        @assert has_name(pkg) || has_uuid(pkg)
        if has_name(pkg) && !has_uuid(pkg)
            for (uuid, name) in ctx.stdlibs
                name == pkg.name && (pkg.uuid = uuid)
            end
        end
        if has_uuid(pkg) && !has_name(pkg)
            haskey(ctx.stdlibs, pkg.uuid) && (pkg.name = ctx.stdlibs[pkg.uuid])
        end
    end
end

# Ensure that all packages are fully resolved
function ensure_resolved(env::EnvCache,
    pkgs::AbstractVector{PackageSpec};
    registry::Bool=false,)::Nothing
    unresolved = Dict{String,Vector{UUID}}()
    for pkg in pkgs
        has_uuid(pkg) && continue
        uuids = UUID[]
        for (name, infos) in env.manifest, info in infos
            name == pkg.name && haskey(info, "uuid") || continue
            uuid = UUID(info["uuid"])
            uuid in uuids || push!(uuids, uuid)
        end
        sort!(uuids, by=uuid -> uuid.value)
        unresolved[pkg.name] = uuids
    end
    isempty(unresolved) && return
    msg = sprint() do io
        println(io, "The following package names could not be resolved:")
        for (name, uuids) in sort!(collect(unresolved), by=lowercase ∘ first)
        print(io, " * $name (")
        if length(uuids) == 0
            what = ["project", "manifest"]
            registry && push!(what, "registry")
            print(io, "not found in ")
            join(io, what, ", ", " or ")
        else
            join(io, uuids, ", ", " or ")
            print(io, " in manifest but not in project")
        end
        println(io, ")")
    end
        print(io, "Please specify by known `name=uuid`.")
    end
    pkgerror(msg)
end

##############
# Registries #
##############

mutable struct RegistrySpec
    name::Union{String,Nothing}
    uuid::Union{UUID,Nothing}
    url::Union{String,Nothing}
    # the path field can be a local source when adding a registry
    # otherwise it is the path where the registry is installed
    path::Union{String,Nothing}
    RegistrySpec(name::String) = RegistrySpec(name = name)
    RegistrySpec(;name=nothing, uuid=nothing, url=nothing, path=nothing) =
        new(name, isa(uuid, String) ? UUID(uuid) : uuid, url, path)
end

const DEFAULT_REGISTRIES =
    RegistrySpec[RegistrySpec(name = "General",
                              uuid = UUID("23338594-aafe-5451-b93e-139f81909106"),
                              url = "https://github.com/JuliaRegistries/General.git")]

function clone_default_registries()
    user_regs = abspath(depots1(), "registries")
    if !ispath(user_regs) || isempty(readdir(user_regs))
        mkpath(user_regs)
        printpkgstyle(stdout, :Cloning, "default registries into $user_regs")
        clone_or_cp_registries(DEFAULT_REGISTRIES)
    end
end

# Return `RegistrySpec`s of each registry in a depot
function collect_registries(depot::String)
    d = joinpath(depot, "registries")
    regs = RegistrySpec[]
    ispath(d) || return regs
    for name in readdir(d)
        file = joinpath(d, name, "Registry.toml")
        if isfile(file)
            registry = read_registry(file)
            verify_registry(registry)
            spec = RegistrySpec(name = registry["name"],
                                uuid = UUID(registry["uuid"]),
                                url = get(registry, "repo", nothing),
                                path = dirname(file))
            push!(regs, spec)
        end
    end
    return regs
end
# Return `RegistrySpec`s of all registries in all depots
function collect_registries()
    isempty(depots()) && return RegistrySpec[]
    return RegistrySpec[r for d in depots() for r in collect_registries(d)]
end

# Hacky way to make e.g. `registry add General` work.
function populate_known_registries_with_urls!(registries::Vector{RegistrySpec})
    known_registries = DEFAULT_REGISTRIES # TODO: Some way to add stuff here?
    for reg in registries, known in known_registries
        if reg.uuid !== nothing
            if reg.uuid === known.uuid
                reg.url = known.url
            end
        elseif reg.name !== nothing
            if reg.name == known.name
                named_regs = filter(r -> r.name == reg.name, known_registries)
                if !all(r -> r.uuid == first(named_regs).uuid, named_regs)
                    pkgerror("multiple registries with name `$(reg.name)`, please specify with uuid.")
                end
                reg.url = known.url
            end
        end
    end
end

# entry point for `registry add`
clone_or_cp_registries(regs::Vector{RegistrySpec}, depot::String=depots1()) =
    clone_or_cp_registries(Context(), regs, depot)
function clone_or_cp_registries(ctx::Context, regs::Vector{RegistrySpec}, depot::String=depots1())
    ctx.preview && (@info("Skipping adding registries in preview mode"); return nothing)
    populate_known_registries_with_urls!(regs)
    for reg in regs
        if reg.path !== nothing && reg.url !== nothing
            pkgerror("ambiguous registry specification; both url and path is set.")
        end
        # clone to tmpdir first
        tmp = mktempdir()
        if reg.path !== nothing # copy from local source
            printpkgstyle(stdout, :Copying, "registry from `$(Base.contractuser(reg.path))`")
            cp(reg.path, tmp; force=true)
        elseif reg.url !== nothing # clone from url
            Base.shred!(LibGit2.CachedCredentials()) do creds
                LibGit2.with(GitTools.clone(reg.url, tmp; header = "registry from $(repr(reg.url))",
                    credentials = creds)) do repo
                end
            end
        else
            pkgerror("no path or url specified for registry")
        end
        # verify that the clone looks like a registry
        if !isfile(joinpath(tmp, "Registry.toml"))
            pkgerror("no `Registry.toml` file in cloned registry.")
        end
        registry = read_registry(joinpath(tmp, "Registry.toml"); cache=false) # don't cache this tmp registry
        verify_registry(registry)
        # copy to `depot`
        # slug = Base.package_slug(UUID(registry["uuid"]))
        regpath = joinpath(depot, "registries", registry["name"]#=, slug=#)
        ispath(dirname(regpath)) || mkpath(dirname(regpath))
        if isdir_windows_workaround(regpath)
            existing_registry = read_registry(joinpath(regpath, "Registry.toml"))
            if registry["uuid"] == existing_registry["uuid"]
                @info("registry `$(registry["name"])` already exist in `$(Base.contractuser(regpath))`.")
            else
                throw(PkgError("registry `$(registry["name"])=\"$(registry["uuid"])\"` conflicts with " *
                    "existing registry `$(existing_registry["name"])=\"$(existing_registry["uuid"])\"`. " *
                    "To install it you can clone it manually into e.g. " *
                    "`$(Base.contractuser(joinpath(depot, "registries", registry["name"]*"-2")))`."))
            end
        else
            cp(tmp, regpath)
            printpkgstyle(stdout, :Added, "registry `$(registry["name"])` to `$(Base.contractuser(regpath))`")
        end
        # Clean up
        Base.rm(tmp; recursive=true, force=true)
    end
    return nothing
end

# path -> (mtime, TOML Dict)
const REGISTRY_CACHE = Dict{String, Tuple{Float64, Dict{String, Any}}}()

function read_registry(reg_file; cache=true)
    t = mtime(reg_file)
    if haskey(REGISTRY_CACHE, reg_file)
        prev_t, registry = REGISTRY_CACHE[reg_file]
        t == prev_t && return registry
    end
    registry = TOML.parsefile(reg_file)
    cache && (REGISTRY_CACHE[reg_file] = (t, registry))
    return registry
end

# verify that the registry looks like a registry
const REQUIRED_REGISTRY_ENTRIES = ("name", "uuid", "repo", "packages") # ??

function verify_registry(registry::Dict{String, Any})
    for key in REQUIRED_REGISTRY_ENTRIES
        haskey(registry, key) || pkgerror("no `$key` entry in `Registry.toml`.")
    end
end

# Search for the input registries among installed ones
function find_installed_registries(needles::Vector{RegistrySpec},
                                   haystack::Vector{RegistrySpec}=collect_registries())
    output = RegistrySpec[]
    for needle in needles
        if needle.name === nothing && needle.uuid === nothing
            pkgerror("no name or uuid specified for registry.")
        end
        found = false
        for candidate in haystack
            if needle.uuid !== nothing
                if needle.uuid == candidate.uuid
                    push!(output, candidate)
                    found = true
                end
            elseif needle.name !== nothing
                if needle.name == candidate.name
                    named_regs = filter(r -> r.name == needle.name, haystack)
                    if !all(r -> r.uuid == first(named_regs).uuid, named_regs)
                        pkgerror("multiple registries with name `$(needle.name)`, please specify with uuid.")
                    end
                    push!(output, candidate)
                    found = true
                end
            end
        end
        if !found
            @info("registry `$(needle.name === nothing ? needle.uuid :
                               needle.uuid === nothing ? needle.name :
                               "$(needle.name)=$(needle.uuid)")` not found.")
        end
    end
    return output
end

# entry point for `registry rm`
function remove_registries(ctx::Context, regs::Vector{RegistrySpec})
    ctx.preview && (@info("skipping removing registries in preview mode"); return nothing)
    for registry in find_installed_registries(regs)
        printpkgstyle(stdout, :Removing, "registry `$(registry.name)` from $(Base.contractuser(registry.path))")
        rm(registry.path; force=true, recursive=true)
    end
    return nothing
end

# entry point for `registry up`
function update_registries(ctx::Context, regs::Vector{RegistrySpec} = collect_registries(depots1()))
    errors = Tuple{String, String}[]
    ctx.preview && (@info("skipping updating registries in preview mode"); return nothing)
    for reg in unique(r -> r.uuid, find_installed_registries(regs))
        if isdir(joinpath(reg.path, ".git"))
            regpath = pathrepr(reg.path)
            printpkgstyle(ctx, :Updating, "registry at " * regpath)
            # Using LibGit2.with here crashes julia when running the
            # tests for PkgDev wiht "Unreachable reached".
            # This seems to work around it.
            local repo
            try
                repo = LibGit2.GitRepo(reg.path)
                if LibGit2.isdirty(repo)
                    push!(errors, (regpath, "registry dirty"))
                    @goto done
                end
                if !LibGit2.isattached(repo)
                    push!(errors, (regpath, "registry detached"))
                    @goto done
                end
                if !("origin" in LibGit2.remotes(repo))
                    push!(errors, (regpath, "origin not in the list of remotes"))
                    @goto done
                end
                branch = LibGit2.headname(repo)
                try
                    GitTools.fetch(repo; refspecs=["+refs/heads/$branch:refs/remotes/origin/$branch"])
                catch e
                    e isa PkgError || rethrow()
                    push!(errors, (reg.path, "failed to fetch from repo"))
                    @goto done
                end
                ff_succeeded = try
                    LibGit2.merge!(repo; branch="refs/remotes/origin/$branch", fastforward=true)
                catch e
                    e isa LibGit2.GitError && e.code == LibGit2.Error.ENOTFOUND || rethrow()
                    push!(errors, (reg.path, "branch origin/$branch not found"))
                    @goto done
                end

                if !ff_succeeded
                    try LibGit2.rebase!(repo, "origin/$branch")
                    catch e
                        e isa LibGit2.GitError || rethrow()
                        push!(errors, (reg.path, "registry failed to rebase on origin/$branch"))
                        @goto done
                    end
                end
                @label done
            finally
                close(repo)
            end
        end
    end
    if !isempty(errors)
        warn_str = "Some registries failed to update:"
        for (reg, err) in errors
            warn_str *= "\n    — $reg — $err"
        end
        @warn warn_str
    end
    UPDATED_REGISTRY_THIS_SESSION[] = true
    return
end

# Lookup package names & uuids in a single pass through registries
function find_registered!(env::EnvCache,
    names::Vector{String},
    uuids::Vector{UUID}=UUID[];
    force::Bool=false,
)::Nothing
    # only look if there's something new to see (or force == true)
    names = filter(name -> !haskey(env.uuids, name), names)
    uuids = filter(uuid -> !haskey(env.paths, uuid), uuids)
    !force && isempty(names) && isempty(uuids) && return

    # since we're looking anyway, look for everything
    save(name::String) =
        name in names || haskey(env.uuids, name) || push!(names, name)
    save(uuid::UUID) =
        uuid in uuids || haskey(env.paths, uuid) || push!(uuids, uuid)

    # lookup any dependency in the project file
    for (name, uuid) in env.project["deps"]
        save(name); save(UUID(uuid))
    end
    # lookup anything mentioned in the manifest file
    for (name, infos) in env.manifest, info in infos
        save(name)
        haskey(info, "uuid") && save(UUID(info["uuid"]))
        haskey(info, "deps") || continue
        for (n, u) in info["deps"]
            save(n); save(UUID(u))
        end
    end
    # if there's still nothing to look for, return early
    isempty(names) && isempty(uuids) && return
    # initialize env entries for names and uuids
    for name in names; env.uuids[name] = UUID[]; end
    for uuid in uuids; env.paths[uuid] = String[]; end
    for uuid in uuids; env.names[uuid] = String[]; end

    # note: empty vectors will be left for names & uuids that aren't found
    clone_default_registries()
    for registry in collect_registries()
        data = read_registry(joinpath(registry.path, "Registry.toml"))
        for (_uuid, pkgdata) in data["packages"]
              uuid = UUID(_uuid)
              name = pkgdata["name"]
              path = abspath(registry.path, pkgdata["path"])
              push!(get!(env.uuids, name, UUID[]), uuid)
              push!(get!(env.paths, uuid, String[]), path)
              push!(get!(env.names, uuid, String[]), name)
        end
    end
    for d in (env.uuids, env.paths, env.names)
        for (k, v) in d
            unique!(v)
        end
    end
end

find_registered!(env::EnvCache, uuids::Vector{UUID}; force::Bool=false)::Nothing =
    find_registered!(env, String[], uuids, force=force)

# Lookup all packages in project & manifest files
find_registered!(env::EnvCache)::Nothing =
    find_registered!(env, String[], UUID[], force=true)

# Get registered uuids associated with a package name
function registered_uuids(env::EnvCache, name::String)::Vector{UUID}
    find_registered!(env, [name], UUID[])
    return unique(env.uuids[name])
end

# Get registered paths associated with a package uuid
function registered_paths(env::EnvCache, uuid::UUID)::Vector{String}
    find_registered!(env, String[], [uuid])
    return env.paths[uuid]
end

#Get registered names associated with a package uuid
function registered_names(env::EnvCache, uuid::UUID)::Vector{String}
    find_registered!(env, String[], [uuid])
    return env.names[uuid]
end

# Determine a single UUID for a given name, prompting if needed
function registered_uuid(env::EnvCache, name::String)::UUID
    uuids = registered_uuids(env, name)
    length(uuids) == 0 && return UUID(zero(UInt128))
    length(uuids) == 1 && return uuids[1]
    choices::Vector{String} = []
    choices_cache::Vector{Tuple{UUID,String}} = []
    for uuid in uuids
        values = registered_info(env, uuid, "repo")
        for value in values
            depot = "(unknown)"
            for d in depots()
                r = joinpath(d, "registries")
                startswith(value[1], r) || continue
                depot = split(relpath(value[1], r), Base.Filesystem.path_separator_re)[1]
                break
            end
            push!(choices, "Registry: $depot - Path: $(value[2])")
            push!(choices_cache, (uuid, value[1]))
        end
    end
    length(choices_cache) == 1 && return choices_cache[1][1]
    if isinteractive()
        # prompt for which UUID was intended:
        menu = RadioMenu(choices)
        choice = request("There are multiple registered `$name` packages, choose one:", menu)
        choice == -1 && return UUID(zero(UInt128))
        env.paths[choices_cache[choice][1]] = [choices_cache[choice][2]]
        return choices_cache[choice][1]
    else
        pkgerror("there are multiple registered `$name` packages, explicitly set the uuid")
    end
end

# Determine current name for a given package UUID
function registered_name(env::EnvCache, uuid::UUID)::String
    names = registered_names(env, uuid)
    length(names) == 0 && return ""
    length(names) == 1 && return names[1]
    values = registered_info(env, uuid, "name")
    name = nothing
    for value in values
        name  == nothing && (name = value[2])
        name != value[2] && pkgerror("package `$uuid` has multiple registered name values: $name, $(value[2])")
    end
    return name
end

# Return most current package info for a registered UUID
function registered_info(env::EnvCache, uuid::UUID, key::String)
    paths = env.paths[uuid]
    isempty(paths) && pkgerror("`$uuid` is not registered")
    values = []
    for path in paths
        info = parse_toml(path, "Package.toml")
        value = get(info, key, nothing)
        push!(values, (path, value))
    end
    return values
end

# Find package by UUID in the manifest file
function manifest_info(env::EnvCache, uuid::UUID)::Union{Dict{String,Any},Nothing}
    uuid in values(env.uuids) || find_registered!(env, [uuid])
    for (name, infos) in env.manifest, info in infos
        haskey(info, "uuid") && uuid == UUID(info["uuid"]) || continue
        return merge!(Dict{String,Any}("name" => name), info)
    end
    return nothing
end

# TODO: redirect to ctx stream
function printpkgstyle(io::IO, cmd::Symbol, text::String, ignore_indent::Bool=false)
    indent = textwidth(string(:Downloaded))
    ignore_indent && (indent = 0)
    printstyled(io, lpad(string(cmd), indent), color=:green, bold=true)
    println(io, " ", text)
end

# TODO: use ctx specific context
function printpkgstyle(ctx::Context, cmd::Symbol, text::String, ignore_indent::Bool=false)
    printpkgstyle(stdout, cmd, text)
end

function pathrepr(path::String)
    # print stdlib paths as @stdlib/Name
    if startswith(path, stdlib_dir())
        path = "@stdlib/" * basename(path)
    end
    return "`" * Base.contractuser(path) * "`"
end

function project_key_order(key::String)
    key == "name"     && return 1
    key == "uuid"     && return 2
    key == "keywords" && return 3
    key == "license"  && return 4
    key == "desc"     && return 5
    key == "deps"     && return 6
    key == "compat"   && return 7
    return 8
end

function write_env(ctx::Context; display_diff=true)
    env = ctx.env
    # load old environment for comparison
    old_env = EnvCache(env.env)
    # update the project file
    project = deepcopy(env.project)
    isempty(project["deps"]) && delete!(project, "deps")
    if !isempty(project) || ispath(env.project_file)
        if display_diff && !(ctx.currently_running_target)
            printpkgstyle(ctx, :Updating, pathrepr(env.project_file))
            Pkg.Display.print_project_diff(ctx, old_env, env)
        end
        if !ctx.preview
            mkpath(dirname(env.project_file))
            open(env.project_file, "w") do io
                TOML.print(io, project, sorted=true, by=key -> (project_key_order(key), key))
            end
        end
    end
    # update the manifest file
    if !isempty(env.manifest) || ispath(env.manifest_file)
        if display_diff && !(ctx.currently_running_target)
            printpkgstyle(ctx, :Updating, pathrepr(env.manifest_file))
            Pkg.Display.print_manifest_diff(ctx, old_env, env)
        end
        manifest = deepcopy(env.manifest)
        uniques = sort!(collect(keys(manifest)), by=lowercase)
        filter!(name -> length(manifest[name]) == 1, uniques)
        uuids = Dict(name => UUID(manifest[name][1]["uuid"]) for name in uniques)
        for (name, infos) in manifest, info in infos
            haskey(info, "deps") || continue
            deps = Dict{String,UUID}(n => UUID(u) for (n, u) in info["deps"])
            all(d in uniques && uuids[d] == u for (d, u) in deps) || continue
            info["deps"] = sort!(collect(keys(deps)))
        end
        if !ctx.preview
            open(env.manifest_file, "w") do io
                TOML.print(io, manifest, sorted=true)
            end
        end
    end
end

end # module


text:
abstract type AbstractFormat end

struct xyzFormat <: AbstractFormat
    fileName::String
end

struct poscarFormat <: AbstractFormat
    fileName::String
end

function format(fileName::String)
    if (subfix = findlast(".", fileName)) !== nothing
        return xyzFormat(fileName)
    else
        return poscarFormat(fileName)
    end
end

ext: jl


text:
""" Draw wiring diagrams using TikZ.
"""
module TikZWiringDiagrams
export to_tikz, layout_to_tikz

using Compat
using DataStructures: OrderedDict
using Match

using ...Syntax: GATExpr, show_latex
using ...WiringDiagrams, ...WiringDiagrams.WiringDiagramSerialization
using ..WiringDiagramLayouts
using ..WiringDiagramLayouts: AbstractVector2D, Vector2D, BoxLayout, box_label,
  wire_label, position, normal, tangent, port_sign, wire_points
import ..WiringDiagramLayouts: svector
import ..TikZ

# Data types
############

""" Internal data type for configurable options of Compose.jl wiring diagrams.
"""
Base.@kwdef struct TikZOptions
  orientation::LayoutOrientation = LeftToRight
  base_unit::String = "4mm"
  labels::Bool = false
  labels_pos::Union{String,Real} = 0.5
  math_mode::Bool = true
  arrowtip::Union{String,Nothing} = nothing
  arrowtip_pos::Union{String,Real} = 0.5
  rounded_boxes::Bool = true
  props::AbstractVector = ["semithick"]
  styles::AbstractDict = Dict()
  libraries::Vector{String} = String[]
  packages::Vector{String} = ["amssymb"]
  used_node_styles::AbstractSet = Set{String}()
end

svector(opts::TikZOptions, args...) = svector(opts.orientation, args...)

# Wiring diagrams
#################

""" Draw a wiring diagram in TikZ.
"""
function to_tikz(args...; orientation::LayoutOrientation=LeftToRight, kw...)
  layout_kw = filter(p -> first(p) ∉ fieldnames(TikZOptions), kw)
  diagram = layout_diagram(args...; orientation=orientation, layout_kw...)

  tikz_kw = filter(p -> first(p) ∈ fieldnames(TikZOptions), kw)
  layout_to_tikz(diagram; orientation=orientation, tikz_kw...)
end

""" Draw a wiring diagram in TikZ using the given layout.
"""
function layout_to_tikz(diagram::WiringDiagram; kw...)::TikZ.Document
  layout_to_tikz(diagram, TikZOptions(; kw...))
end

function layout_to_tikz(diagram::WiringDiagram, opts::TikZOptions)::TikZ.Document
  stmts = tikz_box(diagram, Int[], opts)
  styles, libraries = tikz_styles(opts)
  props = [
    [ # Define a unit length to be used for both TikZ coordinates and lengths.
      # https://tex.stackexchange.com/a/454598
      TikZ.Property("unit length/.code",
        "{{\\newdimen\\$tikz_unit_command}\\setlength{\\$tikz_unit_command}{#1}}"),
      TikZ.Property("unit length", opts.base_unit),
      TikZ.Property("x", "\\$tikz_unit_command"),
      TikZ.Property("y", "\\$tikz_unit_command") ];
    TikZ.as_properties(opts.props);
    [ TikZ.Property("$name/.style", TikZ.as_properties(props))
      for (name, props) in styles ];
  ]
  TikZ.Document(TikZ.Picture(stmts...; props=props);
    libraries=unique!([ "calc"; libraries; opts.libraries ]),
    packages=opts.packages)
end

""" Make TikZ node for a box.
"""
function tikz_box(diagram::WiringDiagram, vpath::Vector{Int}, opts::TikZOptions)
  layout = diagram.value::BoxLayout
  TikZ.Statement[
    TikZ.Node(box_id(vpath),
      props = [TikZ.Property("outer box"); tikz_size(layout.size)],
      coord = tikz_coordinate(layout.position),
    );
    reduce(vcat, [ tikz_box(box(diagram, v), [vpath; v], opts)
                   for v in box_ids(diagram) ], init=[]);
    [ tikz_edge(diagram, wire, opts) for wire in wires(diagram) ];
  ]
end

function tikz_box(box::AbstractBox, vpath::Vector{Int}, opts::TikZOptions)
  layout = box.value::BoxLayout
  style = tikz_node_style(layout, opts)
  TikZ.Statement[
    TikZ.Node(box_id(vpath),
      props = [TikZ.Property(style); tikz_size(layout.size)],
      coord = tikz_coordinate(layout.position),
      content = layout.shape in (:junction, :invisible) ? "" :
        tikz_node_label(layout.value, opts),
    ),
  ]
end

""" Make a TikZ edge/path for a wire.
"""
function tikz_edge(diagram::WiringDiagram, wire::Wire, opts::TikZOptions)::TikZ.Edge
  src, src_angle = tikz_port(diagram, wire.source, opts)
  tgt, tgt_angle = tikz_port(diagram, wire.target, opts)
  exprs, prev_angle = TikZ.PathExpression[ src ], src_angle
  for point in wire_points(wire)
    v = tangent(point)
    push!(exprs, TikZ.PathOperation("to"; props=[
      TikZ.Property("out", string(prev_angle)),
      TikZ.Property("in", string(tikz_angle(-v))),
    ]))
    push!(exprs, tikz_coordinate(position(point)))
    prev_angle = tikz_angle(v)
  end
  push!(exprs, TikZ.PathOperation("to"; props=[
    TikZ.Property("out", string(prev_angle)),
    TikZ.Property("in", string(tgt_angle)),
  ]))
  push!(exprs, tgt)

  # Use source port for wire label, following the Graphviz wiring diagrams.
  src_layout = port_value(diagram, wire.source)
  tgt_layout = port_value(diagram, wire.target)
  wire_node = if opts.labels && src_layout.label_wires && tgt_layout.label_wires
    props = [ TikZ.Property("anchor", tikz_anchor(svector(opts, 0, 1))) ]
    content = tikz_edge_label(src_layout.value, opts)
    node = TikZ.Node(; props=props, content=content)
    string("{", sprint(TikZ.pprint, node), "}")
  else
    nothing
  end
  props = [ TikZ.Property("wire", wire_node) ]
  if !isnothing(opts.arrowtip)
    reversed = src_layout.reverse_wires && tgt_layout.reverse_wires
    push!(props, TikZ.Property(reversed ? "<-" : "->"))
  end

  TikZ.Edge(exprs...; props=props)
end

""" Make TikZ coordinate and angle for a port.
"""
function tikz_port(diagram::WiringDiagram, port::Port, opts::TikZOptions)
  name = box_id(diagram, [port.box])
  parent_box = port.box in outer_ids(diagram) ? diagram : box(diagram, port.box)
  shape = parent_box.value.shape

  x, y = tikz_position(position(port_value(diagram, port)))
  normal_angle = tikz_angle(normal(diagram, port))
  anchor, (x, y) = if shape in (:rectangle, :triangle, :invtriangle,
                                :trapezium, :invtrapezium)
    port_dir = svector(opts, port_sign(diagram, port, opts.orientation), 0)
    e2 = svector(opts, 0, 1)
    (tikz_anchor(port_dir), (e2[1]*x, e2[2]*y))
  elseif shape in (:circle, :ellipse, :junction)
    (normal_angle, (0,0))
  elseif shape == :invisible
    ("center", (0,0))
  else
    # Fallback method. Always works when TikZ follows the layout's box size, but
    # is the least robust to changes in node size.
    ("center", (x,y))
  end

  coord = if x == 0 && y == 0
    "$name.$anchor"
  else
    "\$($name.$anchor)+($x,$y)\$"
  end
  (TikZ.NodeCoordinate(coord), normal_angle)
end

function tikz_node_label(value, opts::TikZOptions)
  if opts.math_mode
    box_label(MIME("text/latex"), value)
  else
    escape_latex(box_label(MIME("text/plain"), value))
  end
end
function tikz_edge_label(value, opts::TikZOptions)
  if opts.math_mode
    wire_label(MIME("text/latex"), value)
  else
    escape_latex(wire_label(MIME("text/plain"), value))
  end
end

""" Escape special LaTeX characters.

Reference: https://tex.stackexchange.com/a/34586/
"""
function escape_latex(s::AbstractString)
  reduce(replace, [
    "\\" => "\\textbackslash",
    "~" => "\\textasciitilde",
    "^" => "\\textasciicircum",
    "&" => "\\&", "%" => "\\%", "\$" => "\\\$", "#" => "\\#",
    "_" => "\\_", "{" => "\\{", "}" => "\\}",
  ], init=s)
end

# TikZ geometry
###############

function tikz_angle(v::AbstractVector2D)
  tikz_number(rad2deg(angle(v[1] - v[2]*im)))
end

function tikz_position(position::AbstractVector2D)::Tuple
  # TikZ's default coordinate system has the positive y-axis going upwards.
  x, y = position
  (tikz_number(x), tikz_number(-y))
end
function tikz_coordinate(position::AbstractVector2D)::TikZ.Coordinate
  TikZ.Coordinate(tikz_position(position)...)
end

function tikz_size(size::AbstractVector2D)::Vector{TikZ.Property}
  width, height = tikz_number(size[1]), tikz_number(size[2])
  if width == 0 && height == 0
    TikZ.Property[]
  elseif width == height
    [ TikZ.Property("minimum size", "$width\\$tikz_unit_command") ]
  else
    [ TikZ.Property("minimum width", "$width\\$tikz_unit_command"),
      TikZ.Property("minimum height", "$height\\$tikz_unit_command") ]
  end
end
const tikz_unit_command = "tikzunit"

tikz_anchor(v::AbstractVector2D) = tikz_anchors[Tuple(v)]

const tikz_anchors = Dict{Tuple{Int,Int},String}(
  (0,0)   => "center",
  (1,0)   => "east",
  (-1,0)  => "west",
  (0,1)   => "south",
  (0,-1)  => "north",
  (1,1)   => "south east",
  (-1,1)  => "south west",
  (1,-1)  => "north east",
  (-1,-1) => "north west",
)

function tikz_number(x::Number; digits=3)::Number
  x = round(x, digits=digits)
  isinteger(x) ? Integer(x) : x
end

# TikZ styles
#############

""" Make TikZ style definitions for wiring diagram.
"""
function tikz_styles(opts::TikZOptions)
  # Box style options.
  styles = OrderedDict(
    style => get(opts.styles, style) do; tikz_node_style(opts, style) end
    for style in sort!([["outer box", "wire"]; collect(opts.used_node_styles)])
  )
  libraries = [ "shapes.geometric" ] # FIXME: Should use library only if needed.

  # Wire style options.
  if opts.labels
    styles["wire"] = vcat(styles["wire"], tikz_decorate_markings([
      "at position $(opts.labels_pos) with {#1}"
    ]))
    push!(libraries, "decorations.markings")
  end
  if !isnothing(opts.arrowtip)
    pos, arrowtip = opts.arrowtip_pos, opts.arrowtip
    merge!(styles, OrderedDict(
      "->" => tikz_decorate_markings([
        "at position $pos with {\\arrow{$arrowtip}}"
      ]),
      "<-" => tikz_decorate_markings([
        "at position $pos with {\\arrow{$arrowtip[reversed]}}"
      ]),
    ))
    append!(libraries, ["arrows.meta", "decorations.markings"])
  end

  (styles, libraries)
end

""" Get default TikZ style for given kind of node.
"""
function tikz_node_style(opts::TikZOptions, name::String)
  rounded = TikZ.Property(opts.rounded_boxes ? "rounded corners" : "sharp corners")
  @match name begin
    "outer box" => [
      TikZ.Property("draw", "none"),
    ]
    "wire" => [
      TikZ.Property("draw"),
    ]
    "box" => [
      TikZ.Property("rectangle"),
      TikZ.Property("draw"), TikZ.Property("solid"), rounded,
    ]
    "circular box" => [
      TikZ.Property("circle"),
      TikZ.Property("draw"), TikZ.Property("solid"),
    ]
    "elliptical box" => [
      TikZ.Property("ellipse"),
      TikZ.Property("draw"), TikZ.Property("solid"),
    ]
    "triangular box" => [
      TikZ.Property("isosceles triangle"),
      TikZ.Property("isosceles triangle stretches"),
      TikZ.Property("shape border rotate", Dict(
        # FIXME: Match.jl doesn't work with enums.
          LeftToRight => "180",
          RightToLeft => "0",
          TopToBottom => "90",
          BottomToTop => "270",
        )[opts.orientation]
      ),
      TikZ.Property("draw"), TikZ.Property("solid"), rounded,
      TikZ.Property("inner sep", "0"),
    ]
    "inverse triangular box" => [
      TikZ.Property("isosceles triangle"),
      TikZ.Property("isosceles triangle stretches"),
      TikZ.Property("shape border rotate", Dict(
          LeftToRight => "0",
          RightToLeft => "180",
          TopToBottom => "270",
          BottomToTop => "90",
        )[opts.orientation]
      ),
      TikZ.Property("draw"), TikZ.Property("solid"), rounded,
      TikZ.Property("inner sep", "0"),
    ]
    "trapezoidal box" => [
      TikZ.Property("trapezium"), TikZ.Property("trapezium angle", "80"),
      TikZ.Property("trapezium stretches body"),
      TikZ.Property("shape border rotate", Dict(
          LeftToRight => "90",
          RightToLeft => "270",
          TopToBottom => "0",
          BottomToTop => "180",
        )[opts.orientation]
      ),
      TikZ.Property("draw"), TikZ.Property("solid"), rounded,
    ]
    "inverse trapezoidal box" => [
      TikZ.Property("trapezium"), TikZ.Property("trapezium angle", "80"),
      TikZ.Property("trapezium stretches body"),
      TikZ.Property("shape border rotate", Dict(
          LeftToRight => "270",
          RightToLeft => "90",
          TopToBottom => "180",
          BottomToTop => "0",
        )[opts.orientation]
      ),
      TikZ.Property("draw"), TikZ.Property("solid"), rounded,
    ]
    "junction" => [
      TikZ.Property("circle"),
      TikZ.Property("draw"), TikZ.Property("fill"),
      TikZ.Property("inner sep", "0"),
    ]
    "variant junction" => [
      TikZ.Property("circle"),
      TikZ.Property("draw"), TikZ.Property("solid"),
      TikZ.Property("inner sep", "0"),
    ]
    "invisible" => [
      TikZ.Property("draw", "none"),
      TikZ.Property("inner sep", "0"),
    ]
    _ => TikZ.Property[]
  end
end

function tikz_decorate_markings(marks::Vector{TikZ.Property})
  [ TikZ.Property("postaction", [ TikZ.Property("decorate") ]),
    TikZ.Property("decoration", [ TikZ.Property("markings"); marks ]) ]
end
function tikz_decorate_markings(marks::Vector{String})
  tikz_decorate_markings([ TikZ.Property("mark", mark) for mark in marks ])
end

""" Get TikZ style for box.
"""
function tikz_node_style(layout::BoxLayout, opts::TikZOptions)
  style = if isnothing(layout.style)
    tikz_shapes[layout.shape]
  else
    replace(string(layout.style), "_" => " ")
  end
  push!(opts.used_node_styles, style)
  style
end

const tikz_shapes = Dict(
  :rectangle => "box",
  :circle => "circular box",
  :ellipse => "elliptical box",
  :triangle => "triangular box",
  :invtriangle => "inverse triangular box",
  :trapezium => "trapezoidal box",
  :invtrapezium => "inverse trapezoidal box",
  :junction => "junction",
  :invisible => "invisible",
)

end


text:
using Test
using GeoParams

@testset "CreepLaw" begin

 #Make sure structs are isbits
 x = LinearViscous()
 @test isbits(x)

 x = PowerlawViscous()
 @test isbits(x)

# This tests the MaterialParameters structure
CharUnits_GEO   =   GEO_units(viscosity=1e19, length=1000km);

# Define a linear viscous creep law ---------------------------------
x1      =   LinearViscous(η=1e18Pa*s)
@test x1.η.val == 1e18

x1_ND   =   LinearViscous(η=1e18Pa*s)
@test  isDimensional(x1_ND)==true
x1_ND = nondimensionalize(x1_ND,CharUnits_GEO)                 # check that we can nondimensionalize all entries within the struct
@test  isDimensional(x1_ND)==false
@test x1_ND.η*1.0==0.1
x1_ND = dimensionalize(x1_ND,CharUnits_GEO)                    # check that we can dimensionalize it again
@test x1_ND.η.val==1e18
x1_ND = nondimensionalize(x1_ND,CharUnits_GEO)

# perform a computation with the viscous creep laws

# Given stress
@test computeCreepLaw_EpsII(1e6Pa, x1, CreepLawVariables())==5e-13/s                # dimensional input
@test computeCreepLaw_EpsII(1e0, x1_ND, CreepLawVariables())==5.0                   # non-dimensional
@test computeCreepLaw_EpsII([1e0; 2.0], x1_ND, CreepLawVariables())==[5.0; 10.0]    # vector input

# Given strainrate
@test computeCreepLaw_TauII(1e-13/s, x1, CreepLawVariables())==1e18*2*1e-13Pa       # dimensional input
@test computeCreepLaw_TauII(1e0, x1_ND, CreepLawVariables())==0.2                   # non-dimensional
@test computeCreepLaw_TauII([1e0; 2.0], x1_ND, CreepLawVariables())==[0.2; 0.4]     # vector input
# -------------------------------------------------------------------

# -------------------------------------------------------------------
# Define powerlaw viscous rheology
x2  =   PowerlawViscous()
x2 = nondimensionalize(x2,CharUnits_GEO)
@test x2.ε0.val ==0.001                                 # powerlaw

# perform computations

# -------------------------------------------------------------------

#include("test_DislocationCreep.jl")

end


text:
export try_chmod

function try_chmod(path, mode; recursive=true)
  try
    chmod(path,mode,recursive=recursive)
  catch
  end
  return
end

function ishidden(filename::AbstractString)
  @static if Sys.isunix()
    s = basename(filename)
    return (!isempty(s) && s[1] == '.')
  else
    attr = ccall((:GetFileAttributesA), stdcall, Cint, (Cstring,),filename)
    return attr & 0x2 > 0
  end
end

function getNewNumInFolder(path)
  if !isdir(path)
    mkpath(path)
    try_chmod(path, 0o777, recursive=true)
    return 1
  end

  files = readdir(path)
  num = 1
  if length(files) > 0
    for i=1:length(files)
      pref, ext = splitext(files[i])
      num_ = tryparse(Int64, pref)
      if num_ != nothing && num_+1>num
        num = num_+1
      end
    end
  end

  return num
end


text:
function initializestate!{T<:Real}(trajectory::DynamicSystemTrajectory1D{T}, state)
    @assert isempty(trajectory.state)
    push!(trajectory.state, state)
    push!(trajectory.value, zero(T))
end

function step!{T<:Real}(trajectory::DynamicSystemTrajectory1D{T}, a::T, t::Int, ω::UnivariateDistribution)
    step!(trajectory, a, t, rand(ω))
end

function step!{T<:Real}(trajectory::DynamicSystemTrajectory1D{T}, a::T, t::Int, w::T)
    # Moves the system from time t to time t+1, and collects data for time t+1
    system = trajectory.system
    @assert 0 <= t < system.T
    x0 = trajectory.state[t+1]
    gainedvalue = system.U(t+1, x0, a, w)
    x1 = system.f(t+1,x0,a,w)
    if t == system.T-1
        gainedvalue += system.Ubar(x1)
    end

    push!(trajectory.control, a)
    push!(trajectory.state, x1)
    push!(trajectory.value, trajectory.value[t+1] + gainedvalue)
end

function initializestate!{T<:Real}(trajectory::DynamicSystemTrajectory{T}, state::Vector{T})
    trajectory.state[:,1] =  state
    trajectory.value[1] = zero(T)
end

function step!{T<:Real}(trajectory::DynamicSystemTrajectory{T}, a::Vector{T},
                        t::Int, W::Tuple)
    step!(trajectory, a, t, rand(ω))
end

function step!{T<:Real}(trajectory::DynamicSystemTrajectory{T}, a,
                        t::Int, w)
    # Moves the system from time t to time t+1, and collects data for time t+1
    system = trajectory.system
    @assert 0 <= t < system.T
    x0 = trajectory.state[:,t+1]
    gainedvalue = system.U(t+1, x0, a, w)
    x1 = system.f(t+1,x0,a,w)
    if t == system.T-1
        gainedvalue += system.Ubar(x1)
    end

    trajectory.control[:,t+1] = a
    trajectory.state[:,t+2] = x1
    trajectory.value[t+2] = trajectory.value[t+1] + gainedvalue
end


text:
# A random ephemeral port. The port should always be free due to AWS Batch multi-mode
# parallel jobs using "awsvpc" networking on containers
const AWS_BATCH_JOB_NODE_PORT = 49152

# The maximum amount of time to listen for worker connections. Note that the manager
# (main node) is started before the workers (other nodes) which allows the manager time to
# initialize before the workers attempt to connect. In a scenario in which a worker fails
# and never connects to the manager this timeout will allow the manager continue with the
# subset of workers which have already connected.
const AWS_BATCH_NODE_TIMEOUT = Minute(5)

# Julia cluster manager for AWS Batch multi-node parallel jobs
# https://docs.aws.amazon.com/batch/latest/userguide/multi-node-parallel-jobs.html
struct AWSBatchNodeManager <: ContainerManager
    num_workers::Int
    timeout::Second  # Duration to wait for workers to initially check-in with the manager

    function AWSBatchNodeManager(; timeout::Period=AWS_BATCH_NODE_TIMEOUT)
        if !haskey(ENV, "AWS_BATCH_JOB_ID") || !haskey(ENV, "AWS_BATCH_JOB_MAIN_NODE_INDEX")
            error(
                "Unable to use $AWSBatchNodeManager outside of a running AWS Batch multi-node parallel job",
            )
        end

        info(LOGGER, "AWS Batch Job ID: $(ENV["AWS_BATCH_JOB_ID"])")

        if ENV["AWS_BATCH_JOB_NODE_INDEX"] != ENV["AWS_BATCH_JOB_MAIN_NODE_INDEX"]
            error("$AWSBatchNodeManager can only be used by the main node")
        end

        # Don't include the manager in the number of workers
        num_workers = parse(Int, ENV["AWS_BATCH_JOB_NUM_NODES"]) - 1

        return new(num_workers, Second(timeout))
    end
end

function Distributed.launch(
    manager::AWSBatchNodeManager, params::Dict, launched::Array, c::Condition
)
    num_workers = manager.num_workers
    connected_workers = 0

    debug(LOGGER, "Awaiting connections")

    # We cannot listen only to the `ENV["AWS_BATCH_JOB_MAIN_NODE_PRIVATE_IPV4_ADDRESS"]` as
    # this variable is not available on the main node. However, since we known the workers
    # will use this IPv4 specific address we'll only listen to IPv4 interfaces.
    server = listen(IPv4(0), AWS_BATCH_JOB_NODE_PORT)
    debug(LOGGER, "Manager accepting worker connections on port $AWS_BATCH_JOB_NODE_PORT")

    # Maintain an internal array of worker configs to allow us to set the ordering
    workers = sizehint!(WorkerConfig[], num_workers)

    listen_task = @async while isopen(server) && connected_workers < num_workers
        # TODO: Potential issue with a random connection consuming a worker slot?
        sock = accept(server)
        connected_workers += 1

        # Receive the job id and node index from the connected worker
        job_id = parse_job_id(readline(sock))
        node_index = parse(Int, match(r"(?<=\#)\d+", job_id).match)

        debug(LOGGER, "Worker connected from node $node_index")

        # Send the cluster cookie and timeout to the worker
        println(sock, "julia_cookie:", cluster_cookie())
        println(sock, "julia_worker_timeout:", Dates.value(Second(manager.timeout)))
        flush(sock)

        # The worker will report it's own address through the socket. Eventually the
        # built in Julia cluster manager code will parse the stream and record the
        # address and port.
        config = WorkerConfig()
        config.io = sock
        config.userdata = (; :job_id => job_id, :node_index => node_index)

        push!(workers, config)
    end

    wait(listen_task, manager.timeout)
    close(server)

    # Note: `launched` is treated as a queue and will have elements removed from it
    # periodically from `addprocs`. By adding all the elements at once we can control the
    # ordering of the workers make it the same as the node index ordering.
    #
    # Note: Julia worker numbers will not match up to the node index of the worker.
    # Primarily this is due to the worker numbers being 1-indexed while nodes are 0-indexed.
    append!(launched, sort!(workers; by=w -> w.userdata.node_index))
    notify(c)

    if connected_workers < num_workers
        warn(
            LOGGER,
            "Only $connected_workers of the $num_workers workers job have reported in",
        )
    else
        debug(LOGGER, "All workers have successfully reported in")
    end
end

function start_batch_node_worker()
    if !haskey(ENV, "AWS_BATCH_JOB_ID") || !haskey(ENV, "AWS_BATCH_JOB_NODE_INDEX")
        error(
            "Unable to start a worker outside of a running AWS Batch multi-node parallel job",
        )
    end

    info(LOGGER, "AWS Batch Job ID: $(ENV["AWS_BATCH_JOB_ID"])")

    # Note: Limiting to IPv4 to match what AWS Batch provides us with for the manager.
    function available_ipv4_msg()
        io = IOBuffer()
        write(io, "Available IPv4 interfaces are:")

        # Include a listing of external IPv4 interfaces and addresses for debugging.
        for i in get_interface_addrs()
            if i.address isa IPv4 && !i.is_internal
                write(io, "\n  $(i.name): inet $(i.address)")
            end
        end

        return String(take!(io))
    end

    # A multi-node parallel job uses the "awsvpc" networking mode which defines "ecs-eth0"
    # in addition to the "eth0" interface. By default Julia tries to use the IP address from
    # "ecs-eth0" which uses a local-link address (169.254.0.0/16) which is unreachable from
    # the manager. Typically this is fixed by specifying the "eth0" IP address as
    # `--bind-to` when starting the Julia worker process.
    opts = Base.JLOptions()
    ip = if opts.bindto != C_NULL
        parse(IPAddr, unsafe_string(opts.bindto))
    else
        getipaddr()
    end

    if is_link_local(ip)
        error(LOGGER) do
            "Aborting due to use of link-local address ($ip) on worker which will be " *
            "unreachable by the manager. Be sure to specify a `--bind-to` address when " *
            "starting Julia. $(available_ipv4_msg())"
        end
    else
        info(LOGGER, "Reporting worker address $ip to the manager")
        debug(LOGGER) do
            available_ipv4_msg()
        end
    end

    # The environmental variable for the main node address is only set within multi-node
    # parallel child nodes and is not present on the main node. See:
    # https://docs.aws.amazon.com/batch/latest/userguide/multi-node-parallel-jobs.html#mnp-env-vars
    manager_ip = parse(IPv4, ENV["AWS_BATCH_JOB_MAIN_NODE_PRIVATE_IPV4_ADDRESS"])

    # Establish a connection to the manager. If the manager is slow to startup the worker
    # will attempt to connect for ~2 minutes.
    manager_connect = retry(
        () -> connect(manager_ip, AWS_BATCH_JOB_NODE_PORT);
        delays=ExponentialBackOff(; n=8, max_delay=30),
        check=(s, e) -> e isa Base.IOError,
    )
    sock = manager_connect()

    # Note: The job ID also contains the node index
    println(sock, "job_id:", ENV["AWS_BATCH_JOB_ID"])
    flush(sock)

    # Retrieve the cluster cookie from the manager
    cookie = parse_cookie(readline(sock))

    # Retrieve the worker timeout as specified in the manager
    timeout = parse_worker_timeout(readline(sock))

    # Hand off control to the Distributed stdlib which will have the worker report an IP
    # address and port at which connections can be established to this worker.
    #
    # The worker timeout needs to be equal to or exceed the amount of time in which the
    # manager waits for workers to report in. The reason for this is that the manager waits
    # to connect to all workers until connecting to any worker. If we use the default
    # timeout then a worker could report in early and self-terminate before the manager
    # connects to that worker. If that scenario occurs then the manager will become stuck
    # during the setup process.
    withenv("JULIA_WORKER_TIMEOUT" => timeout) do
        start_worker(sock, cookie)
    end
end

function parse_job_id(str::AbstractString)
    # Note: Require match on prefix to ensure we are parsing the correct value
    m = match(r"^job_id:([a-z0-9-]{36}\#\d+)", str)

    if m !== nothing
        return m.captures[1]
    else
        error(LOGGER, "Unable to parse job id: $str")
    end
end

function parse_cookie(str::AbstractString)
    # Note: Require match on prefix to ensure we are parsing the correct value
    m = match(r"^julia_cookie:(\w+)", str)

    if m !== nothing
        return m.captures[1]
    else
        error(LOGGER, "Unable to parse cluster cookie: $str")
    end
end

function parse_worker_timeout(str::AbstractString)
    # Note: Require match on prefix to ensure we are parsing the correct value
    m = match(r"^julia_worker_timeout:(\d+)", str)

    if m !== nothing
        return parse(Int, m.captures[1])
    else
        error(LOGGER, "Unable to parse worker timeout: $str")
    end
end


text:
@testset "1553.minimum-number-of-days-to-eat-n-oranges.jl" begin
    @test min_days_1553(159832501) == 28
    @test min_days_1553(82129117) == 28
    @test min_days_1553(1) == 1
    @test min_days_1553(6) == 3
    @test min_days_1553(10) == 4
end


text:
_sm_data = CSV.File(joinpath(@__DIR__, "smdof.csv"))

_sm_heff = CubicSplineInterpolation(
    LinRange(-4.7, 4.1, 276),
    _sm_data.Heff;
    extrapolation_bc = Flat(),
);

_sm_geff = CubicSplineInterpolation(
    LinRange(-4.7, 4.1, 276),
    _sm_data.Geff,
    extrapolation_bc = Flat(),
);

_sm_sqrt_gstar = CubicSplineInterpolation(
    LinRange(-4.7, 4.1, 276),
    _sm_data.SqrtGstar,
    extrapolation_bc = Flat(),
);

sm_heff(T::Real) = _sm_heff(log10(T))
sm_geff(T::Real) = _sm_geff(log10(T))
sm_sqrt_gstar(T::Real) = _sm_sqrt_gstar(log10(T))

sm_entropy_density(T::Real) = 2π^2 / 45 * _sm_heff(log10(T)) * T^3
sm_energy_density(T::Real) = π^2 / 30 * _sm_geff(log10(T)) * T^4

max_stars_repo_name: LoganAMorrison/AsymptoticBoltzmann

max_issues_repo_name: LoganAMorrison/AsymptoticBoltzmann

max_forks_repo_name: LoganAMorrison/AsymptoticBoltzmann


text:
module Mappings

import InverseFunctions: inverse
import Base: HasLength, inv

#export types
export Mapping, AssociativeMap

#export functions
export dom, image, preimage, codom, inverse, inv

abstract type Mapping end
(m::Mapping)(x) = applymap(m, x)
function dom(m::Mapping) end
function image(m::Mapping, x)
    if HasLength(x) || length(x) <=1
        return m(x)
    else
        return Set(unique(m.(x)))
    end
end

image(m::Mapping) = image(m, dom(m))
preimage(m::Mapping) = image(inverse(m))
preimage(m::Mapping, x) = image(inverse(m), x)
codom(m::Mapping) = dom(inverse(m))

#maptypes

#Identity map
# struct IDmap <: Mapping end
# applymap(m::IDmap, x) = x
# one(m::Mapping) = IDmap()

include("AssociativeMaps.jl")

end


text:
mutable struct PylonAcquiredImage{T,N} <: AbstractAcquiredImage{T,N}
    grab_result
    # Inherits behaviour of AbstractPooledDenseArray, by having the same fields
    array::Array{T,N}
    ref_count::Int
    dispose::Function
    function PylonAcquiredImage(grab_result)
        width = Wrapper.get_width(grab_result)
        height = Wrapper.get_height(grab_result)
        buffer = Wrapper.get_buffer(grab_result)
        pixel_type = Wrapper.get_pixel_type(grab_result)
        if Wrapper.is_bgr(pixel_type) || Wrapper.is_rgb(pixel_type)
            samples_per_pixel = Wrapper.samples_per_pixel(pixel_type)
            size = (samples_per_pixel, width, height)
            TPixel = Wrapper.is_bgr(pixel_type) ? BGR : RGB
        else
            size = (width, height)
            TPixel = Gray
        end
        @assert prod(size) == Wrapper.get_image_size(grab_result)
        buffer_array = unsafe_wrap(Array, Ptr{UInt8}(buffer), size)
        image = colorview(TPixel, normedview(buffer_array))
        N = ndims(image)
        new{TPixel{N0f8},N}(grab_result, image, 1, dispose)
    end
end

function dispose(img::PylonAcquiredImage)
    @debug "Releasing $(img.grab_result)"
    Wrapper.release(img.grab_result)
end

id(img::PylonAcquiredImage) = Wrapper.get_id(img.grab_result)
image_number(img::PylonAcquiredImage) = Wrapper.get_image_number(img.grab_result)
timestamp(img::PylonAcquiredImage) = Wrapper.get_time_stamp(img.grab_result)


text:
#Using discrete amounts for molecules might be better than ODEs?
using Catalyst, DiffEqJump, Latexify, Plots

sir_model = @reaction_network begin
    β, S + I --> 2I
    ν, I --> R
end β ν

p     = (0.1/1000,0.01)
u₀    = [999,1,0]
tspan = (0.0,250.0)
prob  = DiscreteProblem(sir_model, u₀, tspan, p)

jump_prob = JumpProblem(sir_model, prob, Direct())

sol = solve(jump_prob, SSAStepper())

plot(sol)

################################################
# Lotka-Volterra Equations
################################################

LV_model = @reaction_network begin
    α,     x --> 2x
    β̄,     x + y --> y
    δ,     x + y --> 2y
    γ,     y --> ∅
end α β̄ δ γ

# odesys = convert(ODESystem, LV_model)
# latexify(LV_model)
# latexify(odesys)

p     = (2.0, 0.02, 0.02, 1.06) #β̄ = β-δ = 0.04-0.02
u₀    = [100, 100]
tspan = (0.0,20.0)

prob  = DiscreteProblem(LV_model, u₀, tspan, p)

jump_prob = JumpProblem(LV_model, prob, RSSA())

sol = solve(jump_prob, SSAStepper())

plot(sol)
plot(sol, vars=(1,2))

######################################################
# Lotka-Volterra Equations (1-D movement on lattice)
######################################################

# Number of lattice points
N = 64

#Parameters are time and rate constant, Variables are amount of predator/prey at each point
@parameters t, h, α, β̄, δ, γ
@variables X[1:N](t) Y[1:N](t)

#Rate to jump between lattice points
rate = 1/h

rxns = []

for i in 1:N

    if i ≠ 1
        push!(rxns, Reaction(rate, [X[i]], [X[i-1]]))
        push!(rxns, Reaction(rate, [Y[i]], [Y[i-1]]))
    end

    if i ≠ N
        push!(rxns, Reaction(rate, [X[i]], [X[i+1]]))
        push!(rxns, Reaction(rate, [Y[i]], [Y[i+1]]))
    end

    push!(rxns, Reaction(α, [X[i]], [X[i]], [1], [2]))
    push!(rxns, Reaction(β̄, [X[i],Y[i]], [Y[i]]))
    push!(rxns, Reaction(δ, [X[i],Y[i]], [Y[i]], [1,1], [2]))
    push!(rxns, Reaction(γ, [Y[i]], nothing))

end

@named LV_spatial = ReactionSystem(rxns, t, [X;Y], [h, α, β̄, δ, γ])

p     = (1/N, 2.0, 0.02, 0.02, 1.06) #β̄ = β-δ = 0.04-0.02
u₀    = zeros(Int64, 2*N)
tspan = (0.0,20.0)

u₀[1] = 100 #100 prey in first lattice
u₀[end] = 100 #100 predators in last lattice

prob  = DiscreteProblem(LV_spatial, u₀, tspan, p)
jump_prob = JumpProblem(LV_spatial, prob, RSSA(),save_positions=(false,false))

sol = solve(jump_prob, SSAStepper(), saveat=0.01)

plot(sol, legend=nothing)

anim = @animate for currState ∈ sol.u
    bar(currState[1:N], alpha=0.5, labels="prey")
    bar!(currState[N+1:end],alpha=0.5, labels="predator")
    ylims!(0,400)
end

gif(anim, "anim.gif", fps = 60)


text:
module RDKafka
using librdkafka_jll

export KafkaProducer,
    KafkaConsumer,
    KafkaClient,
    # produce,
    subscribe,
    seek,
    poll

include("core.jl")

end # module


text:
using LinearAlgebra: Eigen
import LinearAlgebra: eigen, \, det, logdet, inv
import Base: +

"""
    eigen(K::AbstractKroneckerProduct)

Wrapper around `eigen` from the `LinearAlgebra` package. Performs Eigenvalue decompositon
on the matrices of a `AbstractKroneckerProduct` instances and returns an
`Eigen` type. The functions `size`, `\\`, `inv`, `det`,
and `logdet` are overloaded to efficiently work with this type.
"""
function eigen(K::AbstractKroneckerProduct)
    squarecheck(K)
    A, B = getmatrices(K)
    A_λ, A_Γ = eigen(A)
    B_λ, B_Γ = eigen(B)
    return Eigen(kron(A_λ, B_λ), kronecker(A_Γ, B_Γ))
end

+(E::Eigen, B::UniformScaling) = Eigen(E.values .+ B.λ, E.vectors)
+(A::UniformScaling, E::Eigen) = E + A

"""
    function collect(E::Eigen{<:Number, <:Number, <:AbstractKroneckerProduct})

Collects eigenvalue decomposition of a `AbstractKroneckerProduct` type into a
matrix.
"""
function collect(E::Eigen{<:Number, <:Number, <:AbstractKroneckerProduct})
    λ, Γ = E
    return Γ * Diagonal(λ) * Γ'
end

function \(E::Eigen{<:Number, <:Number, <:AbstractKroneckerProduct}, v::AbstractVector{<:Number})
    λ, Γ = E
    return Γ * (Diagonal(λ) \ (Γ' * v))
end

"""
    det(K::Eigen)

Compute the determinant of the eigenvalue decomp of aKronecker product.
"""
det(E::Eigen{<:Number, <:Number, <:AbstractKroneckerProduct}) = prod(E.values)

"""
    logdet(K::Eigen)

Compute the logarithm of the determinant of the eigenvalue decomp of aKronecker
product.
"""
logdet(E::Eigen{<:Number, <:Number, <:AbstractKroneckerProduct}) = sum(log, E.values)

"""
    inv(K::Eigen)

Compute the inverse of the eigenvalue decomp of aKronecker product. Returns
another type of `Eigen`.
"""
inv(E::Eigen{<:Number, <:Number, <:AbstractKroneckerProduct}) = Eigen(inv.(E.values), E.vectors)


text:
export IAAFT
"""
    IAAFT(M = 100, tol = 1e-6, W = 75)

An iteratively adjusted amplitude-adjusted-fourier-transform surrogate[^SchreiberSchmitz1996].

IAAFT surrogate have the same linear correlation, or periodogram, and also
preserves the amplitude distribution of the original data, but are improved relative
to AAFT through iterative adjustment (which runs for a maximum of `M` steps).
During the iterative adjustment, the periodograms of the original signal and the
surrogate are coarse-grained and the powers are averaged over `W` equal-width
frequency bins. The iteration procedure ends when the relative deviation
between the periodograms is less than `tol` (or when `M` is reached).

IAAFT, just as AAFT, can be used to test the null hypothesis that the data
come from a monotonic nonlinear transformation of a linear Gaussian process.

[^SchreiberSchmitz1996]: T. Schreiber; A. Schmitz (1996). "Improved Surrogate Data for Nonlinearity Tests". [Phys. Rev. Lett. 77 (4)](https://journals.aps.org/prl/abstract/10.1103/PhysRevLett.77.635)
"""
struct IAAFT <: Surrogate
    M::Int
    tol::Real
    W::Int

    function IAAFT(;M::Int = 100, tol::Real = 1e-6, W::Int = 75)
        new(M, tol, W)
    end
end

Base.show(io::IO, x::IAAFT) = print(io, "IAAFT(M=$(x.M), tol=$(x.tol), W=$(x.W))")

function surrogenerator(x, method::IAAFT)
    # Pre-plan Fourier transforms
    forward = plan_rfft(x)
    inverse = plan_irfft(forward*x, length(x))

    # Pre-compute stuff that can be used for different surrogate realizations
    m = mean(x)
    x_sorted = sort(x)
    𝓕 = forward*(x .- m)
    r_original = abs.(𝓕)

    # Coarse-grain the periodograms when comparing them between iterations.
    px = DSP.periodogram(x)
    range = LinRange(0.0, 0.5, method.W)
    px_binned = interp(px.freq, px.power, range)

    # These are updated during iteration procedure
    𝓕new = Vector{Complex{Float64}}(undef, length(𝓕))
    𝓕sorted = Vector{Complex{Float64}}(undef, length(𝓕))
    ϕsorted = Vector{Complex{Float64}}(undef, length(𝓕))

    init = (forward = forward, inverse = inverse, m = m, 𝓕 = 𝓕, r_original = r_original,
            px_binned = px_binned, range = range, x_sorted = x_sorted,
            𝓕new = 𝓕new, 𝓕sorted =  𝓕sorted, ϕsorted = ϕsorted)

    return SurrogateGenerator(method, x, init)
end

function (sg::SurrogateGenerator{<:IAAFT})()
    init_fields = (:forward, :inverse, :m, :𝓕, :r_original,
                    :px_binned, :range,
                    :x_sorted,
                    :𝓕new, :𝓕sorted, :ϕsorted)
    forward, inverse, m, 𝓕, r_original,
        px_binned, range,
        x_sorted,
        𝓕new, 𝓕sorted, ϕsorted = getfield.(Ref(sg.init), init_fields)

    x = sg.x
    M = sg.method.M
    tol = sg.method.tol

    # Keep track of difference between periodograms between iterations
    diffs = zeros(Float64, 2)

    # RANK ORDERING.
    # Create some Gaussian noise, and find the indices that sorts it with
    # increasing amplitude. Then sort the original time series according to
    # the indices rendering the Gaussian noise sorted.
    n = length(x)
    g = rand(Normal(), n)
    ts_sorted = x[sortperm(g)]

    # The surrogate
    s = Vector{Float64}(undef, n)

    iter = 1
    success = false
    while iter <= M
        # Take the Fourier transform of `ts_sorted` and get the phase angles of
        # the resulting complex numbers.
        𝓕sorted .= forward * ts_sorted
        ϕsorted .= angle.(𝓕sorted)

        # The new spectrum preserves the amplitudes of the Fourier transform of
        # the original time series, but randomises the phases (because the
        # phases are derived from the *randomly sorted* version of the original
        # time series).
        𝓕new .= r_original .* exp.(ϕsorted .* 1im)

        # Now, let the surrogate time series be the values of the original time
        # series, but sorted according to the new spectrum. The shuffled series
        # is generated by taking the inverse Fourier transform of the spectrum
        # consisting of the original amplitudes, but having randomised phases.
        s .= real.(inverse * 𝓕new) # ifft normalises by default

        # map original values onto shuffle
        s[sortperm(s)] = x_sorted
        ts_sorted .= s

        # Convergence check on periodogram
        ps = DSP.periodogram(s)
        ps_binned = interp(ps.freq, ps.power, range)

        if iter == 1
            diffs[1] = sum((px_binned[2] .- ps_binned[2]).^2) / sum(px_binned[2].^2)
        else
            diffs[2] = sum((px_binned[2] .- ps_binned[2]).^2) / sum(px_binned[2].^2)
            abs(diffs[1] - diffs[2]) < tol ? break : diffs[1] = copy(diffs[2])
        end

        iter += 1
    end

    return s
end


text:
#Fit N-mixture model

using Random, Distributions, StatsFuns, Optim, NLSolversBase, ForwardDiff
using LinearAlgebra

using Unmarked

Random.seed!(123);

#Simulate data
N = 4000;
J = 5;
z = rand(Poisson(5),N);
r = 0.1;

y = Array{Int64,2}(undef, N, J);

for i = 1:N
  for j = 1:J
    p = 1 - (1-r)^z[i];
    y[i,j] = rand(Binomial(1, p));
  end
end

um = UmData(y)

@time mod = rn(@formula(λ~1), @formula(p~1), um, K=50)

#Display
truth = [log(5), logit(r)];
pnames = ["β[1]", "β[2]"];
cnames = ["param" "truth" "estimate" "se"];
[cnames; pnames truth coef(mod) stderror(mod)]

max_stars_repo_name: kenkellner/julia-hme

max_issues_repo_name: kenkellner/julia-hme

max_forks_repo_name: kenkellner/julia-hme


text:
# runtests

for s in readdir("."; join=true, sort=true)
    f, e = splitext(s)
    if e == ".pdf" || e == ".vtu" || e == ".pvd" || e == ".res"
        try
            rm(s)
        catch
            @warn "Failed to remove $s"
        end
    end
end

for s in [
    "clamp_cyl_transient_time_step",
    "spherical_cap_expl_examples",
    "spherical_cap_expl_KE",
    "plate_expl_examples",
    "angle_bar_examples",
    "cracked_half_cylinder_examples",
    ]
    include(s * ".jl")
end


text:
using POMDPs
using POMDPPolicies
using POMDPSimulators
using POMDPModels
using BenchmarkTools
using Random

mdps = [LegacyGridWorld(terminals = Set()),
        SimpleGridWorld(terminate_in = Set()),
       ]

for m in mdps
    @show typeof(m)
    policy = RandomPolicy(m, rng=MersenneTwister(7))
    rosim = RolloutSimulator(max_steps=10_000, rng=MersenneTwister(2))
    @btime simulate($rosim, $m, $policy)
end


text:
function transporter_density_to_monomer_uptake_sites(V_c::Vector{Float64}, ρ_closure::Matrix{Float64}, Min_gen_time::Vector{Float64}, Gram_stain::Vector{String})
    gmax = log(2)./Min_gen_time
    n_0 = cell_volume_to_cell_number_density(V_c, gmax, Gram_stain)
    r_p = 1e-9                    # 1 μm
    r_c = (3*V_c/(4*pi)).^(1/3)
    N_SB = zeros(size(ρ_closure,1), size(ρ_closure,2))
    for i in 1:size(V_c,1)
        N_SB[:,i] = n_0[i]*4*r_c[i]^2/(r_p^2)*ρ_closure[:,i]/12.011
    end
    N_SB[N_SB.<1e-12].=0.0
    return N_SB
end

function specific_reference_affinity(V_c::Vector{Float64}, ρ_p::Matrix{Float64}, D_S::Vector{Float64})
    # changed N_p (the number of transporters) to ρ_p = N_p*4*pi*r_p^2/(4*pi*r_c^2) (the transporter density on the cell surface)
    V_S = 1800.0                   # k2p, [s]
    N_A = 6.022e23                # Avogrado constant [1/mol]
    r_p = 1e-9                    # porter radius, 1 μm
    r_c = (3*V_c/(4*pi)).^(1/3)   # cell radius
    #
    p_inv = zeros(size(ρ_p,1), size(ρ_p,2))
    for i in 1:size(ρ_p,2)
        p_inv[:,i] = @. (4*ρ_p[:,i]*r_c[i]^2/r_p + pi*r_c[i])/(4*ρ_p[:,i]*r_c[i]^2/r_p)
    end
    #
    K_SC_0 = zeros(size(ρ_p,1), size(ρ_p,2))
    for i in 1:size(ρ_p,2)
        K_SC_0[:,i] = @. ((V_S*ρ_p[:,i]*r_c[i])/(pi*D_S*r_p^2*N_A))*p_inv[:,i]
    end
    #
    any(x->x==true, isnan.(K_SC_0)) ? throw(DomainError("NaN in DEBmicroTrait.ECA_kinetics!"))  : return K_SC_0
end

function transporters_closure(ρ_p::Vector{Float64}, genome_distr::Matrix{Float64})
    closure   = zeros(size(genome_distr))
    for i in 1:size(closure,2)
        closure[:,i] = ρ_p[i]*genome_distr[:,i]./sum(genome_distr[:,i])
    end
    closure[closure.==0.0].=1e-8
    return closure
end

max_stars_repo_name: giannamars/DEBMicroTrait

max_issues_repo_name: giannamars/DEBMicroTrait

max_forks_repo_name: giannamars/DEBMicroTrait


text:
function hash_subtree(node::Union{IOTreeRoot, IOTreeNode}, h::UInt64)
    for c in node.children
        h = hash(c, h)
    end
    h
end
hash_subtree(leaf::IOTreeLeaf, h::UInt64) = hash(leaf.value, h)
Base.hash(node::Union{IOTreeRoot, IOTreeNode, IOTreeLeaf}, h::UInt64) = hash_subtree(node, hash(node.mask, h))

subtree_equal(lhs::IOTreeObject, rhs::IOTreeObject) = false
subtree_equal(lhs::IOTreeLeaf, rhs::IOTreeLeaf) = lhs.value == rhs.value
function subtree_equal(lhs::Union{IOTreeRoot, IOTreeNode}, rhs::Union{IOTreeRoot, IOTreeNode})
    for (lc, rc) in zip(lhs.children, rhs.children)
        (lc == rc) || return false
    end
    return true
end

function Base.:(==)(lhs::Union{IOTreeNode, IOTreeLeaf}, rhs::Union{IOTreeNode, IOTreeLeaf})
    return false
end
function Base.:(==)(lhs::IOTreeNode, rhs::IOTreeNode)
    (lhs.mask == rhs.mask) && subtree_equal(lhs, rhs)
end
function Base.:(==)(lhs::IOTreeLeaf, rhs::IOTreeLeaf)
    (lhs.mask == rhs.mask) && lhs.value == rhs.value
end

function fold!(tree)
    hashes = map(x->x[1]=>hash_subtree(x[2], UInt64(0)), enumerate(children(tree)))
    sort!(hashes, by = x->(x[2],x[1]))
    i,j = 1,2
    while j <= length(hashes)
        if hashes[i][2] == hashes[j][2] &&
              subtree_equal(children(tree)[hashes[i][1]], children(tree)[hashes[j][1]])
            # Empty the second one, but don't delete it for now to
            # keep indices valid
            old_mask = children(tree)[hashes[j][1]].mask
            children(tree)[hashes[j][1]] = IOTreeNode(zeros_mask)
            # Merge the second one into the first
            new_mask = children(tree)[hashes[i][1]].mask .| old_mask
            children(tree)[hashes[i][1]] =
              isa(children(tree)[hashes[i][1]], IOTreeLeaf) ?
                IOTreeLeaf(new_mask, children(tree)[hashes[i][1]].value) :
                IOTreeNode(new_mask, children(tree)[hashes[i][1]])
        else
            i = j
        end
        j += 1
    end
    filter!(c->!isa(c,IOTreeNode)||!isempty(c.children), tree.children)
    for c in children(tree)
        isa(c, IOTreeNode) && fold!(c)
    end
end