text:
export build_tarballs, autobuild, print_artifacts_toml, build, get_meta_json
import GitHub: gh_get_json, DEFAULT_API
import SHA: sha256, sha1
using Pkg.TOML, Dates, UUIDs
using RegistryTools
import LibGit2
import PkgLicenses

const DEFAULT_JULIA_VERSION_SPEC = "1.0"

const BUILD_HELP = (
    """
    Usage: build_tarballs.jl [target1,target2,...] [--help]
                             [--verbose] [--debug]
                             [--deploy] [--deploy-bin] [--deploy-jll]
                             [--register] [--meta-json]

    Options:
        targets             By default `build_tarballs.jl` will build a tarball
                            for every target within the `platforms` variable.
                            To override this, pass in a list of comma-separated
                            target triplets for each target to be built.  Note
                            that this can be used to build for platforms that
                            are not listed in the 'default list' of platforms
                            in the build_tarballs.jl script.

        --verbose           This streams compiler output to stdout during the
                            build which can be very helpful for finding bugs.
                            Note that it is colorized if you pass the
                            --color=yes option to julia, see examples below.

        --debug=<mode>      This causes a failed build to drop into an
                            interactive shell for debugging purposes.  `<mode>`
                            can be one of `error`, `begin` or `end`.  `error`
                            drops you into the interactive shell only when there
                            is an error during the build, this is the default
                            when no mode is specified.  `begin` forces an error
                            at the beginning of the build, before any command in
                            the script is run.  `end` forces an error at the end
                            of the build script, useful to debug a successful
                            build for which the auditor would fail.

        --deploy=<repo>     Deploy binaries and JLL wrapper code to a github
                            release of an autogenerated repository.  Uses
                            `github.com/JuliaBinaryWrappers/<name>_jll.jl` by
                            default, unless `<repo>` is set, in which case it
                            should be set as `<owner>/<name>_jll.jl`.  Setting
                            this option is equivalent to setting `--deploy-bin`
                            and `--deploy-jll`.  If `<repo>` is set to "local"
                            then nothing will be uploaded, but JLL packages
                            will still be written out to `~/.julia/dev/`.

        --deploy-bin=<repo> Deploy just the built binaries

        --deploy-jll=<repo> Deploy just the JLL code wrappers

        --register=<depot>  Register into the given depot.  If no path is
                            given, defaults to `~/.julia`.  Registration
                            requires deployment of the JLL wrapper code, so
                            so using `--register` without `--deploy` or the
                            more specific `--deploy-jll` is an error.

        --meta-json         Output a JSON representation of the given build
                            instead of actually building.  Note that this can
                            (and often does) output multiple JSON objects for
                            multiple platforms, multi-stage builds, etc...

        --help              Print out this message.

    Examples:
        julia --color=yes build_tarballs.jl --verbose
            This builds all tarballs, with colorized output.

        julia build_tarballs.jl x86_64-linux-gnu,i686-linux-gnu
            This builds two tarballs for the two platforms given, with a
            minimum of output messages.
    """
)

"""
    build_tarballs(ARGS, src_name, src_version, sources, script, platforms,
                   products, dependencies; kwargs...)

This should be the top-level function called from a `build_tarballs.jl` file.
It takes in the information baked into a `build_tarballs.jl` file such as the
`sources` to download, the `products` to build, etc... and will automatically
download, build and package the tarballs, generating a `build.jl` file when
appropriate.

Generally, `ARGS` should be the top-level Julia `ARGS` command-line arguments
object.  `build_tarballs` does some rudimentary parsing of the arguments. To
see what it can do, you can call it with `--help` in the `ARGS` or see the
[Command Line](@ref) section in the manual.

The `kwargs` are passed on to [`autobuild`](@ref), see there for a list of
supported ones. A few additional keyword arguments are accept:

* `julia_compat` can be set to a version string which is used to set the
  supported Julia version in the `[compat]` section of the `Project.toml` of
  the generated JLL package. The default value is `"1.0"`.

* `lazy_artifacts` sets whether the artifacts should be lazy.

* `init_block` may be set to a string containing Julia code; if present, this
  code will be inserted into the initialization path of the generated JLL
  package. This can for example be used to invoke an initialization API of a
  shared library.

!!! note

    The `init_block` keyword argument is experimental and may be removed
    in a future version of this package. Please use it sparingly.
"""
function build_tarballs(ARGS, src_name, src_version, sources, script,
                        platforms, products, dependencies;
                        julia_compat::String = DEFAULT_JULIA_VERSION_SPEC,
                        kwargs...)
    @nospecialize
    # See if someone has passed in `--help`, and if so, give them the
    # assistance they so clearly long for
    if "--help" in ARGS
        println(BUILD_HELP)
        return nothing
    end

    # XXX: These are needed as long as we support old-style sources and
    # dependencies.  Raise a warning for now, deprecate in BB 0.3+
    sources = coerce_source.(sources)
    dependencies = coerce_dependency.(dependencies)

    # Do not clobber caller's ARGS
    ARGS = deepcopy(ARGS)

    # This sets whether we should build verbosely or not
    verbose = check_flag!(ARGS, "--verbose")

    # This sets whether we drop into a debug shell on failure or not
    debug, debug_mode = extract_flag!(ARGS, "--debug", "error")

    # Are we skipping building and just outputting JSON?
    meta_json, meta_json_file = extract_flag!(ARGS, "--meta-json")

    # This sets whether we are going to deploy our binaries/wrapper code to GitHub releases
    deploy, deploy_repo = extract_flag!(ARGS, "--deploy", "JuliaBinaryWrappers/$(src_name)_jll.jl")
    deploy_bin, deploy_bin_repo = extract_flag!(ARGS, "--deploy-bin", "JuliaBinaryWrappers/$(src_name)_jll.jl")
    deploy_jll, deploy_jll_repo = extract_flag!(ARGS, "--deploy-jll", "JuliaBinaryWrappers/$(src_name)_jll.jl")

    # Resolve deploy settings
    if deploy
        deploy_bin = true
        deploy_jll = true
        deploy_bin_repo = deploy_repo
        deploy_jll_repo = deploy_repo
    elseif deploy_bin # make sure bin repo and jll repo match
        deploy_jll_repo = deploy_bin_repo
    elseif deploy_jll
        deploy_bin_repo = deploy_jll_repo
    elseif deploy_bin && deploy_jll
        if deploy_bin_repo != deploy_jll_repo
            error("Binaries and JLLs must be deployed to the same repositories")
        end
    end

    # This sets whether we are going to register, and if so, which
    register, register_path = extract_flag!(ARGS, "--register", Pkg.depots1())
    if register && !deploy_jll
        error("Cannot register without deploying!")
    end
    if register && deploy_jll_repo == "local"
        error("Cannot register with a local deployment!")
    end

    if deploy_bin || deploy_jll
        code_dir = joinpath(Pkg.devdir(), "$(src_name)_jll")

        # Shove them into `kwargs` so that we are conditionally passing them along
        kwargs = (; kwargs..., code_dir = code_dir)
    end

    # If --meta-json was passed, error out if any confusing options were passed
    meta_json_stream = nothing
    if meta_json
        if deploy || deploy_bin || deploy_jll
            error("Cannot specify --deploy* with --meta-json!")
        end
        if register
            error("Cannot specify --register with --meta-json!")
        end
        if debug
            error("Cannot specify --debug with --meta-json!")
        end

        # Otherwise, check to see if we're spitting it out to stdout or a file:
        if meta_json_file === nothing
            meta_json_stream = stdout
        else
            meta_json_stream = open(meta_json_file, "a")
        end
    end

    # If the user passed in a platform (or a few, comma-separated) on the
    # command-line, use that instead of our default platforms
    if length(ARGS) > 0
        parse_platform(p::AbstractString) = p == "any" ? AnyPlatform() : parse(Platform, p; validate_strict=true)
        platforms = parse_platform.(split(ARGS[1], ","))
    end

    # Check to make sure we have the necessary environment stuff
    if deploy_bin || deploy_jll
        # Check to see if we've already got a wrapper package within the Registry,
        # choose a version number that is greater than anything else existent.
        build_version = get_next_wrapper_version(src_name, src_version)
        if verbose
            @info("Building and deploying version $(build_version) to $(deploy_jll_repo)")
        end
        tag = "$(src_name)-v$(build_version)"

        # We need to make sure that the JLL repo at least exists, so that we can deploy binaries to it
        # even if we're not planning to register things to it today.
        if deploy_jll_repo != "local"
            init_jll_package(src_name, code_dir, deploy_jll_repo)
        end
    end

    # Modify script for debugging
    if debug
        if debug_mode == "begin"
            script = "false\n" * script
        elseif debug_mode == "end"
            script = script * "\nfalse"
        end
    end

    args = (
        # Source information
        src_name,
        src_version,
        sources,

        # Build script
        script,

        # Platforms to build for
        platforms,

        # Products we're expecting
        products,

        # Dependencies that must be downloaded
        dependencies,
    )
    extra_kwargs = extract_kwargs(kwargs, (:lazy_artifacts, :init_block))

    if meta_json_stream !== nothing
        # If they've asked for the JSON metadata, by all means, give it to them!
        dict = get_meta_json(args...; extra_kwargs..., julia_compat=julia_compat)
        println(meta_json_stream, JSON.json(dict))

        if meta_json_stream !== stdout
            close(meta_json_stream)
        end

        build_output_meta = Dict()
    else
        # Build the given platforms using the given sources
        build_output_meta = autobuild(
            # Controls output product placement, mount directory placement, etc...
            pwd(),

            args...;

            # Flags
            verbose=verbose,
            debug=debug,
            kwargs...,
        )
    end

    if deploy_jll
        if verbose
            @info("Committing and pushing $(src_name)_jll.jl wrapper code version $(build_version)...")
        end

        # For deploy discard build-only dependencies
        # and make sure we get a `Vector{Dependency}`
        dependencies = Dependency[dep for dep in dependencies if is_runtime_dependency(dep)]

        # The location the binaries will be available from
        bin_path = "https://github.com/$(deploy_jll_repo)/releases/download/$(tag)"
        build_jll_package(src_name, build_version, sources, code_dir, build_output_meta,
                          dependencies, bin_path; verbose=verbose, extra_kwargs...)
        if deploy_jll_repo != "local"
            push_jll_package(src_name, build_version; code_dir=code_dir, deploy_repo=deploy_jll_repo)
        end
        if register
            if verbose
                @info("Registering new wrapper code version $(build_version)...")
            end

            register_jll(src_name, build_version, dependencies, julia_compat;
                         deploy_repo=deploy_jll_repo, code_dir=code_dir, extra_kwargs...)
        end
    end

    if deploy_bin && deploy_bin_repo != "local"
        # Upload the binaries
        if verbose
            @info("Deploying binaries to release $(tag) on $(deploy_bin_repo) via `ghr`...")
        end
        upload_to_github_releases(deploy_bin_repo, tag, joinpath(pwd(), "products"); verbose=verbose)
    end

    return build_output_meta
end

function check_flag!(ARGS, flag)
    flag_present = flag in ARGS
    filter!(x -> x != flag, ARGS)
    return flag_present
end

function extract_flag!(ARGS, flag, val = nothing)
    for f in ARGS
        if f == flag || startswith(f, string(flag, "="))
            # Check if it's just `--flag` or if it's `--flag=foo`
            if f != flag
                val = split(f, '=')[2]
            end

            # Drop this value from our ARGS
            filter!(x -> x != f, ARGS)
            return (true, val)
        end
    end
    return (false, val)
end

"""
    get_compilers_versions(; compilers = [:c])

Return the script string that is used to print the versions of the given `compilers`.
"""
function get_compilers_versions(; compilers = [:c])
    output =
        """
        set -x
        """
    if :c in compilers
        output *=
            """
            cc --version
            c++ --version
            gcc --version
            g++ --version
            clang --version
            clang++ --version
            objc --version
            f77 --version
            gfortran --version
            ld -v
            """
    end
    if :go in compilers
        output *=
            """
            go version
            """
    end
    if :rust in compilers
        output *=
            """
            rustc --version
            rustup --version
            cargo --version
            """
    end
    return output
end

function upload_to_github_releases(repo, tag, path; gh_auth=Wizard.github_auth(;allow_anonymous=false),
                                   attempts::Int = 3, verbose::Bool = false)
    for attempt in 1:attempts
        try
            ghr() do ghr_path
                run(`$ghr_path -u $(dirname(repo)) -r $(basename(repo)) -t $(gh_auth.token) $(tag) $(path)`)
            end
            return
        catch
            if verbose
                @info("`ghr` upload step failed, beginning attempt #$(attempt)...")
            end
        end
    end
    error("Unable to upload $(path) to GitHub repo $(repo) on tag $(tag)")
end

function get_next_wrapper_version(src_name, src_version)
    # If src_version already has a build_number, just return it immediately
    if src_version.build != ()
        return src_version
    end
    ctx = Pkg.Types.Context()

    # Force-update the registry here, since we may have pushed a new version recently
    update_registry(ctx, devnull)

    # If it does, we need to bump the build number up to the next value
    build_number = 0
    if any(isfile(joinpath(p, "Package.toml")) for p in Pkg.Operations.registered_paths(ctx, jll_uuid("$(src_name)_jll")))
        # Find largest version number that matches ours in the registered paths
        versions = VersionNumber[]
        for path in Pkg.Operations.registered_paths(ctx, jll_uuid("$(src_name)_jll"))
            append!(versions, RegistryTools.Compress.load_versions(joinpath(path, "Versions.toml")))
        end
        versions = filter(v -> (v.major == src_version.major) &&
                            (v.minor == src_version.minor) &&
                            (v.patch == src_version.patch) &&
                            (v.build isa Tuple{<:UInt}), versions)
        # Our build number must be larger than the maximum already present in the registry
        if !isempty(versions)
            build_number = first(maximum(versions).build) + 1
        end
    end

    # Construct build_version (src_version + build_number)
    build_version = VersionNumber(src_version.major, src_version.minor,
                         src_version.patch, src_version.prerelease, (build_number,))
end

function _registered_packages(registry_url::AbstractString)
    tmp_dir = mktempdir()
    atexit(() -> rm(tmp_dir; force = true, recursive = true))
    registry_dir = joinpath(tmp_dir, "REGISTRY")
    LibGit2.clone(registry_url, registry_dir)
    registry = TOML.parsefile(joinpath(registry_dir, "Registry.toml"))
    packages = Vector{String}(undef, 0)
    for p in registry["packages"]
        push!(packages, p[2]["name"])
    end
    rm(tmp_dir; force = true, recursive = true)
    return packages
end

function _package_is_registered(registry_url::AbstractString,
                                package::AbstractString)
    registered_packages = _registered_packages(registry_url)
    return package in registered_packages
end

is_yggdrasil() = get(ENV, "YGGDRASIL", "false") == "true"
# Use an Azure Pipelines environment variable to get the current commit hash
yggdrasil_head() = get(ENV, "BUILD_SOURCEVERSION", "")

function register_jll(name, build_version, dependencies, julia_compat;
                      deploy_repo="JuliaBinaryWrappers/$(name)_jll.jl",
                      code_dir=joinpath(Pkg.devdir(), "$(name)_jll"),
                      gh_auth=Wizard.github_auth(;allow_anonymous=false),
                      gh_username=gh_get_json(DEFAULT_API, "/user"; auth=gh_auth)["login"],
                      lazy_artifacts::Bool=false,
                      kwargs...)
    if !Base.isidentifier(name)
        error("Package name \"$(name)\" is not a valid identifier")
    end
    # Calculate tree hash of wrapper code
    wrapper_tree_hash = bytes2hex(Pkg.GitTools.tree_hash(code_dir))

    # Use RegistryTools to push up a new `General` branch with this JLL package registered within it
    # TODO: Update our fork periodically from upstream `General`.
    cache = RegistryTools.RegistryCache(joinpath(Pkg.depots1(), "registries_binarybuilder"))
    registry_url = "https://$(gh_username):$(gh_auth.token)@github.com/JuliaRegistries/General"
    cache.registries[registry_url] = Base.UUID("23338594-aafe-5451-b93e-139f81909106")
    project = Pkg.Types.Project(build_project_dict(name, build_version, dependencies, julia_compat; lazy_artifacts=lazy_artifacts))
    errors = setdiff(RegistryTools.registrator_errors, [:version_less_than_all_existing])
    reg_branch = RegistryTools.register(
        "https://github.com/$(deploy_repo).git",
        project,
        wrapper_tree_hash;
        registry=registry_url,
        cache=cache,
        push=true,
        checks_triggering_error = errors,
    )
    if haskey(reg_branch.metadata, "error")
        @error(reg_branch.metadata["error"])
    else
        upstream_registry_url = "https://github.com/JuliaRegistries/General"
        name_jll = "$(name)_jll"
        if _package_is_registered(upstream_registry_url, name_jll)
            pr_title = "New version: $(name_jll) v$(build_version)"
        else
            pr_title = "New package: $(name_jll) v$(build_version)"
        end
        # Open pull request against JuliaRegistries/General
        body = """
        Autogenerated JLL package registration

        * Registering JLL package $(basename(deploy_repo))
        * Repository: https://github.com/$(deploy_repo)
        * Version: v$(build_version)
        """
        if is_yggdrasil()
            body *= """
                    * Revision on Yggdrasil: https://github.com/JuliaPackaging/Yggdrasil/commit/$(yggdrasil_head())
                    """
        end
        params = Dict(
            "base" => "master",
            "head" => "$(reg_branch.branch)",
            "maintainer_can_modify" => true,
            "title" => pr_title,
            "body" => body,
        )
        Wizard.create_or_update_pull_request("JuliaRegistries/General", params; auth=gh_auth)
    end
end

function get_meta_json(
                   src_name::AbstractString,
                   src_version::VersionNumber,
                   sources::Vector{<:AbstractSource},
                   script::AbstractString,
                   platforms::Vector,
                   products::Vector{<:Product},
                   dependencies::Vector{<:AbstractDependency};
                   julia_compat::String = DEFAULT_JULIA_VERSION_SPEC,
                   lazy_artifacts::Bool = false,
                   init_block::String = "")

    dict = Dict(
        "name" => src_name,
        "version" => "v$(src_version)",
        "sources" => sources,
        "script" => script,
        "products" => products,
        "dependencies" => dependencies,
        "julia_compat" => julia_compat,
        "lazy_artifacts" => lazy_artifacts,
        "init_block" => init_block,
    )
    # Do not write the list of platforms when building only for `AnyPlatform`
    if platforms != [AnyPlatform()]
        dict["platforms"] = triplet.(platforms)
    end
    return dict
end

"""
    autobuild(dir::AbstractString, src_name::AbstractString,
              src_version::VersionNumber, sources::Vector,
              script::AbstractString, platforms::Vector,
              products::Vector, dependencies::Vector;
              verbose = false, debug = false,
              skip_audit = false, ignore_audit_errors = true,
              autofix = true, code_dir = nothing,
              meta_json_file = nothing, require_license = true, kwargs...)

Runs the boiler plate code to download, build, and package a source package
for a list of platforms.  This method takes a veritable truckload of arguments,
here are the relevant actors, broken down in brief:

* `dir`: the root of the build; products will be placed within `dir`/products,
   and mountpoints will be placed within `dir`/build/.

* `src_name`: the name of the source package being built and will set the name
   of the built tarballs.

* `src_version`: the version of the source package.

* `platforms`: a list of platforms to build for.

* `sources`: a vector of all sources to download and unpack before building
  begins, as [`AbstractSource`](@ref)s.

* `script`: a string representing a shell script to run as the build.

* `products`: the list of `Product`s which shall be built.

* `dependencies`: a vector of JLL dependency packages as
  [`AbstractDependency`](@ref) that should be installed before building begins.

* `verbose`: Enable verbose mode.  What did you expect?

* `debug`: cause a failed build to drop into an interactive shell so that
   the build can be inspected easily.

* `skip_audit`: disable the typical audit that occurs at the end of a build.

* `ignore_audit_errors`: do not kill a build even if a problem is found.

* `autofix`: give `BinaryBuilder` permission to automatically fix issues it
   finds during audit passes.  Highly recommended.

* `code_dir`: sets where autogenerated JLL packages will be put.

* `require_license` enables a special audit pass that requires licenses to be
   installed by all packages.
"""
function autobuild(dir::AbstractString,
                   src_name::AbstractString,
                   src_version::VersionNumber,
                   sources::Vector{<:AbstractSource},
                   script::AbstractString,
                   platforms::Vector,
                   products::Vector{<:Product},
                   dependencies::Vector{<:AbstractDependency};
                   verbose::Bool = false,
                   debug::Bool = false,
                   skip_audit::Bool = false,
                   ignore_audit_errors::Bool = true,
                   autofix::Bool = true,
                   code_dir::Union{String,Nothing} = nothing,
                   require_license::Bool = true,
                   kwargs...)
    @nospecialize

    # If we're on CI and we're not verbose, schedule a task to output a "." every few seconds.
    # Exclude GitHub Actions, it wouldn't time out
    if (haskey(ENV, "TRAVIS") || (haskey(ENV, "CI") && !haskey(ENV, "GITHUB_ACTIONS"))) && !verbose
        run_travis_busytask = true
        travis_busytask = @async begin
            # Don't let Travis think we're asleep...
            @info("Brewing a pot of coffee for Travis...")
            while run_travis_busytask
                sleep(4)
                print(".")
            end
        end
    end

    # This is what we'll eventually return
    @info("Building for $(join(triplet.(platforms), ", "))")
    build_output_meta = Dict()

    # Resolve dependencies into PackageSpecs now, ensuring we have UUIDs for all deps
    all_resolved, dependencies = resolve_jlls(dependencies, outs=(verbose ? stdout : devnull))
    if !all_resolved
        error("Invalid dependency specifications!")
    end

    # If the user passed in a src_version with a build number, bail out
    if src_version.build != ()
        error("Will not build with a `src_version` that has a build number already specified!")
    end

    # We must prepare our sources.  Download them, hash them, etc...
    source_files = download_source.(sources; verbose=verbose)

    # Our build products will go into ./products
    out_path = joinpath(dir, "products")
    try mkpath(out_path) catch; end

    for platform in sort(collect(platforms), by = triplet)
        # We build in a platform-specific directory
        build_path = joinpath(dir, "build", triplet(platform))
        mkpath(build_path)

        shards = choose_shards(platform; extract_kwargs(kwargs, (:preferred_gcc_version,:preferred_llvm_version,:bootstrap_list,:compilers))...)
        concrete_platform = get_concrete_platform(platform, shards)

        prefix = setup_workspace(
            build_path,
            source_files,
            concrete_platform,
            default_host_platform;
            verbose=verbose,
        )
        host_artifact_paths = setup_dependencies(prefix, Pkg.Types.PackageSpec[getpkg(d) for d in dependencies if is_host_dependency(d)], default_host_platform; verbose=verbose)
        target_artifact_paths = setup_dependencies(prefix, Pkg.Types.PackageSpec[getpkg(d) for d in dependencies if is_target_dependency(d)], concrete_platform; verbose=verbose)

        # Create a runner to work inside this workspace with the nonce built-in
        ur = preferred_runner()(
            prefix.path;
            cwd = "/workspace/srcdir",
            platform = concrete_platform,
            verbose = verbose,
            workspaces = [
                joinpath(prefix, "metadir") => "/meta",
            ],
            compiler_wrapper_dir = joinpath(prefix, "compiler_wrappers"),
            src_name = src_name,
            shards = shards,
            extract_kwargs(kwargs, (:preferred_gcc_version,:preferred_llvm_version,:compilers,:allow_unsafe_flags,:lock_microarchitecture))...,
        )

        # Set up some bash traps
        trapper_wrapper = """
        # Stop if we hit any errors.
        set -e

        # If we're running as `bash`, then use the `DEBUG` and `ERR` traps
        if [ \$(basename \$0) = "bash" ]; then
            trap "RET=\\\$?; \\
                  trap - DEBUG INT TERM ERR EXIT; \\
                  set +e +x; \\
                  auto_install_license; \\
                  save_env; \\
                  exit \\\$RET" \\
                EXIT

            trap "RET=\\\$?; \\
                  trap - DEBUG INT TERM ERR EXIT; \\
                  set +e +x; \\
                  echo Previous command \\\$! exited with \\\$RET >&2; \\
                  save_env; \\
                  exit \\\$RET" \\
                INT TERM ERR

            # Start saving everything into our history
            trap save_history DEBUG
        else
            # If we're running in `sh` or something like that, we need a
            # slightly slimmer set of traps. :(
            trap "RET=\\\$?; \\
                  echo Previous command exited with \\\$RET >&2; \\
                  set +e +x; \\
                  save_env; \\
                  exit \\\$RET" \\
                EXIT INT TERM
        fi

        $(script)
        """

        dest_prefix = Prefix(BinaryBuilderBase.destdir(prefix.path, concrete_platform))
        did_succeed = with_logfile(dest_prefix, "$(src_name).log") do io
            # Let's start the presentations with BinaryBuilder.jl
            write(io, "BinaryBuilder.jl version: $(get_bb_version())\n\n")
            # Get the list of compilers...
            compilers = extract_kwargs(kwargs, (:compilers,))
            # ...because we want to log all their versions.  However, we don't
            # want this to be shown in the console, so we first run this without
            # teeing to stdout
            run(ur, `/bin/bash -l -c $(get_compilers_versions(; compilers...))`, io;
                verbose = verbose, tee_stream = devnull)
            # Run the build script
            run(ur, `/bin/bash -l -c $(trapper_wrapper)`, io; verbose=verbose)
        end
        if !did_succeed
            if debug
                @warn("Build failed, launching debug shell")
                run_interactive(ur, `/bin/bash -l -i`)
            end
            msg = "Build for $(src_name) on $(triplet(platform)) did not complete successfully\n"
            error(msg)
        end

        # Run an audit of the prefix to ensure it is properly relocatable
        if !skip_audit
            audit_result = audit(dest_prefix, src_name;
                                 platform=platform, verbose=verbose,
                                 has_csl = any(getname.(dependencies) .== "CompilerSupportLibraries_jll"),
                                 autofix=autofix, require_license=require_license)
            if !audit_result && !ignore_audit_errors
                msg = replace("""
                Audit failed for $(dest_prefix.path).
                Address the errors above to ensure relocatability.
                To override this check, set `ignore_audit_errors = true`.
                """, '\n' => ' ')
                error(strip(msg))
            end
        end

        # Finally, error out if something isn't satisfied
        unsatisfied_so_die = false
        for p in products
            if platform isa AnyPlatform && !(p isa FileProduct)
                # `AnyPlatform` is by design platform-independent, so we allow
                # only `FileProduct`s.
                error("Cannot have $(typeof(p)) for AnyPlatform")
            end
            if !satisfied(p, dest_prefix; verbose=verbose, platform=platform)
                if !verbose
                    # If we never got a chance to see the verbose output, give it here:
                    locate(p, dest_prefix; verbose=true, platform=platform)
                end
                @error("Built $(src_name) but $(variable_name(p)) still unsatisfied:")
                unsatisfied_so_die = true
            end
        end
        if unsatisfied_so_die
            error("Cannot continue with unsatisfied build products!")
        end

        # We also need to capture some info about each product
        products_info = Dict{Product,Any}()
        for p in products
            product_path = locate(p, dest_prefix; platform=platform)
            products_info[p] = Dict("path" => relpath(product_path, dest_prefix.path))
            if p isa LibraryProduct || p isa FrameworkProduct
                products_info[p]["soname"] = something(
                    Auditor.get_soname(product_path),
                    basename(product_path),
                )
            end
        end

        # Unsymlink all the deps from the dest_prefix
        cleanup_dependencies(prefix, host_artifact_paths)
        cleanup_dependencies(prefix, target_artifact_paths)

        # Search for dead links in dest_prefix; raise warnings about them.
        Auditor.warn_deadlinks(dest_prefix.path)

        # Cull empty directories, for neatness' sake, unless auditing is disabled
        if !skip_audit
            for (root, dirs, files) = walkdir(dest_prefix.path; topdown=false)
                # We do readdir() here because `walkdir()` does not do a true in-order traversal
                if isempty(readdir(root))
                    rm(root)
                end
            end
        end

        # Compress log files
        compress_dir(joinpath(dest_prefix.path, "logs"), verbose=verbose)

        # Once we're built up, go ahead and package this dest_prefix out
        tarball_path, tarball_hash, git_hash = package(
            dest_prefix,
            joinpath(out_path, src_name),
            src_version;
            platform=platform,
            verbose=verbose,
            force=true,
        )

        build_output_meta[platform] = (
            tarball_path,
            tarball_hash,
            git_hash,
            products_info,
        )

        # Destroy the workspace, taking care to make sure that we don't run into any
        # permissions errors while we do so.
        Base.Filesystem.prepare_for_deletion(prefix.path)
        rm(prefix.path; recursive=true)

        # If the whole build_path is empty, then remove it too.  If it's not, it's probably
        # because some other build is doing something simultaneously with this target, and we
        # don't want to mess with their stuff.
        if isempty(readdir(build_path))
            rm(build_path; recursive=true)
        end
    end

    if (haskey(ENV, "TRAVIS") || (haskey(ENV, "CI") && !haskey(ENV, "GITHUB_ACTIONS"))) && !verbose
        run_travis_busytask = false
        wait(travis_busytask)
        println()
    end

    # Return our product hashes
    return build_output_meta
end

function download_github_release(download_dir, repo, tag; gh_auth=Wizard.github_auth(), verbose::Bool=false)
    release = gh_get_json(DEFAULT_API, "/repos/$(repo)/releases/tags/$(tag)", auth=gh_auth)
    assets = [a for a in release["assets"] if endswith(a["name"], ".tar.gz")]

    for asset in assets
        if verbose
            @info("Downloading $(asset["name"])")
        end
        download(asset["browser_download_url"], joinpath(download_dir, asset["name"]))
    end
    return assets
end

function init_jll_package(name, code_dir, deploy_repo;
                          gh_auth = Wizard.github_auth(;allow_anonymous=false),
                          gh_username = gh_get_json(DEFAULT_API, "/user"; auth=gh_auth)["login"])
    try
        # This throws if it does not exist
        GitHub.repo(deploy_repo; auth=gh_auth)
    catch e
        # If it doesn't exist, create it.
        # check whether gh_org might be a user, not an organization.
        gh_org = dirname(deploy_repo)
        isorg = GitHub.owner(gh_org; auth=gh_auth).typ == "Organization"
        owner = GitHub.Owner(gh_org, isorg)
        @info("Creating new wrapper code repo at https://github.com/$(deploy_repo)")
        try
            GitHub.create_repo(owner, basename(deploy_repo), Dict("license_template" => "mit", "has_issues" => "false"); auth=gh_auth)
        catch create_e
            # If creation failed, it could be because the repo was created in the meantime.
            # Check for that; if it still doesn't exist, then freak out.  Otherwise, continue on.
            try
                GitHub.repo(deploy_repo; auth=gh_auth)
            catch
                rethrow(create_e)
            end
        end
    end

    if !isdir(code_dir)
        # If it does exist, clone it down:
        @info("Cloning wrapper code repo from https://github.com/$(deploy_repo) into $(code_dir)")
        Wizard.with_gitcreds(gh_username, gh_auth.token) do creds
            LibGit2.clone("https://github.com/$(deploy_repo)", code_dir; credentials=creds)
        end
    else
        # Otherwise, hard-reset to latest main:
        repo = LibGit2.GitRepo(code_dir)
        Wizard.with_gitcreds(gh_username, gh_auth.token) do creds
            LibGit2.fetch(repo; credentials=creds)
        end
        main_branch = LibGit2.lookup_branch(repo, "origin/main", true)
        # Starting from 2020-10-01 GitHub uses `main` as the default name of the
        # main branch of a repository, but it used to use `master`
        if isnothing(main_branch)
            main_branch = LibGit2.lookup_branch(repo, "origin/master", true)
            remote_branch = "master"
        else
            remote_branch = "main"
        end
        origin_main_oid = LibGit2.GitHash(main_branch)
        LibGit2.reset!(repo, origin_main_oid, LibGit2.Consts.RESET_HARD)
        if string(LibGit2.head_oid(repo)) != string(origin_main_oid) || remote_branch == "master"
            LibGit2.branch!(repo, "main", string(origin_main_oid); force=true)
        end
    end
end

# rebuild_jll_package is not called from anywhere in BinaryBuilder,
# but rather from JuliaPackaging/Yggdrasil/.ci/register_package.jl
function rebuild_jll_package(obj::Dict;
                             download_dir = nothing,
                             upload_prefix = nothing,
                             build_version = nothing,
                             gh_org::String = "JuliaBinaryWrappers",
                             verbose::Bool = false,
                             lazy_artifacts::Bool = false,
                             from_scratch::Bool = true)
    if build_version === nothing
        build_version = BinaryBuilder.get_next_wrapper_version(obj["name"], obj["version"])
    end
    if download_dir === nothing
        download_dir = mktempdir()
        repo = "$(gh_org)/$(obj["name"])_jll.jl"
        tag = "$(obj["name"])-v$(build_version)"
        download_github_release(download_dir, repo, tag; verbose=verbose)
        upload_prefix = "https://github.com/$(repo)/releases/download/$(tag)"
    elseif upload_prefix === nothing
        error("If download_dir is specified, you must specify upload_prefix as well!")
    end

    return rebuild_jll_package(
        obj["name"],
        build_version,
        obj["sources"],
        obj["platforms"],
        obj["products"],
        obj["dependencies"],
        download_dir,
        upload_prefix;
        verbose=verbose,
        lazy_artifacts = lazy_artifacts,
        julia_compat = get(obj, "julia_compat", DEFAULT_JULIA_VERSION_SPEC),
        init_block = get(obj, "init_block", ""),
        from_scratch = from_scratch,
    )
end

function rebuild_jll_package(name::String, build_version::VersionNumber, sources::Vector,
                             platforms::Vector, products::Vector, dependencies::Vector,
                             download_dir::String, upload_prefix::String;
                             code_dir::String = joinpath(Pkg.devdir(), "$(name)_jll"),
                             verbose::Bool = false, from_scratch::Bool = true,
                             kwargs...)
    # We're going to recreate "build_output_meta"
    build_output_meta = Dict()

    # Then generate a JLL package for each platform
    downloaded_files = readdir(download_dir)
    for platform in sort(collect(platforms), by = triplet)
        # Find the corresponding tarball:
        tarball_idx = findfirst([occursin(".$(triplet(platform)).", f) for f in downloaded_files])

        # No tarball matching the given platform...
        if tarball_idx === nothing
            # ..but wait, maybe the tarball still uses the os version number for
            # FreeBSD or macOS?
            for (isos, try_os_version) in ((Sys.isfreebsd, "11.1"), (Sys.isapple, "14"))
                if isos(platform) && os_version(platform) === nothing
                    tmp_platform = deepcopy(platform)
                    tmp_platform["os_version"] = try_os_version
                    tarball_idx = findfirst([occursin(".$(triplet(tmp_platform)).", f) for f in downloaded_files])
                end
            end
        end

        # Ok, really no tarball matching the given platform
        if tarball_idx === nothing
            error("Incomplete JLL release!  Could not find tarball for $(triplet(platform))")
        end
        tarball_path = joinpath(download_dir, downloaded_files[tarball_idx])

        # Begin reconstructing all the information we need
        tarball_hash = open(tarball_path, "r") do io
            bytes2hex(sha256(io))
        end

        # Unpack the tarball into a new location, calculate the git hash and locate() each product;
        mktempdir() do dest_prefix
            unpack(tarball_path, dest_prefix)

            git_hash = Base.SHA1(Pkg.GitTools.tree_hash(dest_prefix))
            if verbose
                @info("Calculated git tree hash $(bytes2hex(git_hash.bytes)) for $(basename(tarball_path))")
            end

            # Determine locations of each product
            products_info = Dict{Product,Any}()
            for p in products
                product_path = locate(p, Prefix(dest_prefix); platform=platform, verbose=verbose, skip_dlopen=true)
                if product_path === nothing
                    error("Unable to locate $(p) within $(dest_prefix) for $(triplet(platform))")
                end
                products_info[p] = Dict("path" => relpath(product_path, dest_prefix))
                if p isa LibraryProduct || p isa FrameworkProduct
                    products_info[p]["soname"] = something(
                        Auditor.get_soname(product_path),
                        basename(product_path),
                    )
                end
            end

            # Store all this information within build_output_meta:
            build_output_meta[platform] = (
                joinpath(upload_prefix, downloaded_files[tarball_idx]),
                tarball_hash,
                git_hash,
                products_info,
            )

            # Override read-only permissions before cleaning-up the directory
            Base.Filesystem.prepare_for_deletion(dest_prefix)
        end
    end

    # If `from_scratch` is set (the default) we clear out any old crusty code
    # before generating our new, pristine, JLL package within it.  :)
    if from_scratch
        rm(joinpath(code_dir, "src"); recursive=true, force=true)
        rm(joinpath(code_dir, "Artifacts.toml"); force=true)
    end

    # Finally, generate the full JLL package
    build_jll_package(name, build_version, sources, code_dir, build_output_meta,
                      dependencies, upload_prefix; verbose=verbose,
                      kwargs...)
end

function build_jll_package(src_name::String,
                           build_version::VersionNumber,
                           sources::Vector,
                           code_dir::String,
                           build_output_meta::Dict,
                           dependencies::Vector,
                           bin_path::String;
                           verbose::Bool = false,
                           julia_compat::String = DEFAULT_JULIA_VERSION_SPEC,
                           lazy_artifacts::Bool = false,
                           init_block = "")
    if !Base.isidentifier(src_name)
        error("Package name \"$(src_name)\" is not a valid identifier")
    end
    # Make way, for prince artifacti
    mkpath(joinpath(code_dir, "src", "wrappers"))

    # Drop BuildDependency objects
    dependencies = Dependency[d for d in dependencies if is_runtime_dependency(d)]

    platforms = keys(build_output_meta)
    products_info = Dict{Product,Any}
    for platform in sort(collect(platforms), by = triplet)
        if verbose
            @info("Generating jll package for $(triplet(platform)) in $(code_dir)")
        end

        # Extract this platform's information.  Each of these things can be platform-specific
        # (including the set of products!) so be general here.
        tarball_name, tarball_hash, git_hash, products_info = build_output_meta[platform]

        # Add an Artifacts.toml
        artifacts_toml = joinpath(code_dir, "Artifacts.toml")
        download_info = Tuple[
            (joinpath(bin_path, basename(tarball_name)), tarball_hash),
        ]
        if platform isa AnyPlatform
            # AnyPlatform begs for a platform-independent artifact
            bind_artifact!(artifacts_toml, src_name, git_hash; download_info=download_info, force=true, lazy=lazy_artifacts)
        else
            bind_artifact!(artifacts_toml, src_name, git_hash; platform=platform, download_info=download_info, force=true, lazy=lazy_artifacts)
        end

        # Generate the platform-specific wrapper code
        open(joinpath(code_dir, "src", "wrappers", "$(triplet(platform)).jl"), "w") do io
            println(io, "# Autogenerated wrapper script for $(src_name)_jll for $(triplet(platform))")
            if !isempty(products_info)
                println(io, """
                export $(join(sort(variable_name.(first.(collect(products_info)))), ", "))
                """)
            end
            for dep in dependencies
                println(io, "using $(getname(dep))")
            end

            # Generate header definitions like `find_artifact_dir()`
            println(io, "JLLWrappers.@generate_wrapper_header($(repr(src_name)))")

            # Next, begin placing products
            function global_declaration(p::LibraryProduct, p_info::Dict)
                return "JLLWrappers.@declare_library_product($(variable_name(p)), $(repr(p_info["soname"])))"
            end
            global_declaration(p::FrameworkProduct, p_info::Dict) = global_declaration(p.libraryproduct, p_info)

            function global_declaration(p::ExecutableProduct, p_info::Dict)
                vp = variable_name(p)
                # An executable product's public interface is a do-block wrapper function
                return "JLLWrappers.@declare_executable_product($(variable_name(p)))"
            end

            function global_declaration(p::FileProduct, p_info::Dict)
                return "JLLWrappers.@declare_file_product($(variable_name(p)))"
            end

            # Create relative path mappings that are compile-time constant, and mutable
            # mappings that are initialized by __init__() at load time.
            for (p, p_info) in sort(products_info)
                println(io, global_declaration(p, p_info))
            end

            print(io, """
            function __init__()
                JLLWrappers.@generate_init_header($(join(getname.(dependencies), ", ")))
            """)

            for (p, p_info) in sort(products_info)
                vp = variable_name(p)

                if Sys.iswindows(platform)
                    # `dlopen` on Windows isn't occasionally happy to see
                    # forward slashes in the path:
                    # https://github.com/JuliaPackaging/BinaryBuilder.jl/issues/941.
                    # Workaround the issue by normalising the path separator to
                    # the backslash.
                    p_info["path"] = replace(p_info["path"], "/" => "\\")
                end

                if p isa LibraryProduct || p isa FrameworkProduct
                    println(io, """
                        JLLWrappers.@init_library_product(
                            $(vp),
                            $(repr(p_info["path"])),
                            $(BinaryBuilderBase.dlopen_flags_str(p)),
                        )
                    """)
                elseif p isa ExecutableProduct
                    println(io, """
                        JLLWrappers.@init_executable_product(
                            $(vp),
                            $(repr(p_info["path"])),
                        )
                    """)
                elseif p isa FileProduct
                    println(io, """
                        JLLWrappers.@init_file_product(
                            $(vp),
                            $(repr(p_info["path"])),
                        )
                    """)
                end
            end
            println(io, "    JLLWrappers.@generate_init_footer()")

            if !isempty(init_block)
                print(io, """
                    $(init_block)
                """)
            end

            print(io, """
            end  # __init__()
            """)
        end
    end

    # Generate target-demuxing main source file.
    open(joinpath(code_dir, "src", "$(src_name)_jll.jl"), "w") do io
        print(io, """
        # Use baremodule to shave off a few KB from the serialized `.ji` file
        baremodule $(src_name)_jll
        using Base
        using Base: UUID
        """)
        if lazy_artifacts
            println(io, "using LazyArtifacts")
        end
        print(io, """
        import JLLWrappers

        JLLWrappers.@generate_main_file_header($(repr(src_name)))
        JLLWrappers.@generate_main_file($(repr(src_name)), $(repr(jll_uuid("$(src_name)_jll"))))
        end  # module $(src_name)_jll
        """)
    end

    print_source(io, s::ArchiveSource) = println(io, "* compressed archive: ", s.url, " (SHA256 checksum: `", s.hash,"`)")
    print_source(io, s::GitSource) =     println(io, "* git repository: ", s.url, " (revision: `", s.hash,"`)")
    print_source(io, s::FileSource) =    println(io, "* file: ", s.url, " (SHA256 checksum: `", s.hash,"`)")
    function print_source(io, s::DirectorySource)
        print(io, "* files in directory, relative to originating `build_tarballs.jl`: ")
        if is_yggdrasil()
            println(io, "[`", s.path, "`](https://github.com/JuliaPackaging/Yggdrasil/tree/", yggdrasil_head(), "/", ENV["PROJECT"], "/", basename(s.path), ")")
        else
            println(io, "`", s.path, "`")
        end
    end
    function print_jll(io, dep)
        depname = getname(dep)
        if is_yggdrasil()
            # In this case we can easily add a direct link to the repo
            println(io, "* [`", depname, "`](https://github.com/JuliaBinaryWrappers/", depname, ".jl)")
        else
            println(io, "* `", depname, ")`")
        end
    end
    print_product(io, p::Product) = println(io, "* `", typeof(p), "`: `", variable_name(p), "`")
    # Add a README.md
    open(joinpath(code_dir, "README.md"), "w") do io
        print(io,
              """
              # `$(src_name)_jll.jl` (v$(build_version))

              This is an autogenerated package constructed using [`BinaryBuilder.jl`](https://github.com/JuliaPackaging/BinaryBuilder.jl).""")
        if is_yggdrasil()
            print(io, " The originating [`build_tarballs.jl`](https://github.com/JuliaPackaging/Yggdrasil/blob/$(yggdrasil_head())/$(ENV["PROJECT"])/build_tarballs.jl) script can be found on [`Yggdrasil`](https://github.com/JuliaPackaging/Yggdrasil/), the community build tree.  If you have any issue, please report it to the Yggdrasil [bug tracker](https://github.com/JuliaPackaging/Yggdrasil/issues).")
        end
        println(io)
        println(io)
        println(io,"For more details about JLL packages and how to use them, see `BinaryBuilder.jl` [documentation](https://juliapackaging.github.io/BinaryBuilder.jl/dev/jll/).")
        println(io)
        if length(sources) > 0
            # `sources` can be empty, and it is for some HelloWorld examples
            println(io, """
                        ## Sources

                        The tarballs for `$(src_name)_jll.jl` have been built from these sources:""")
            println(io)
            print_source.(Ref(io), sources)
            println(io)
        end
        println(io, """
                    ## Platforms

                    `$(src_name)_jll.jl` is available for the following platforms:
                    """)
        for p in sort(collect(platforms), by = triplet)
            println(io, "* `", p, "` (`", triplet(p), "`)")
        end
        if length(dependencies) > 0
            println(io)
            println(io, """
                        ## Dependencies

                        The following JLL packages are required by `$(src_name)_jll.jl`:""")
            println(io)
            print_jll.(Ref(io), sort(dependencies, by = getname))
        end
        if length(keys(products_info)) > 0
            println(io)
            println(io, """
                        ## Products

                        The code bindings within this package are autogenerated from the following `Products`:
                        """)
            for (p, _) in sort(products_info)
                print_product(io, p)
            end
        end
    end

    # Add before the license a note about to what files this applies
    license = if isfile(joinpath(code_dir, "LICENSE"))
        # In most cases we have a file called `LICENSE`...
        strip(read(joinpath(code_dir, "LICENSE"), String))
    else
        # ...but sometimes this is missing.
        strip("MIT License\n\nCopyright (c) $(year(now()))\n" * PkgLicenses.readlicense("MIT"))
    end
    note_lines = split("""
                       The Julia source code within this repository (all files under `src/`) are
                       released under the terms of the MIT \"Expat\" License, the text of which is
                       included below.  This license does not apply to the binary package wrapped by
                       this Julia package and automatically downloaded by the Julia package manager
                       upon installing this wrapper package.  The binary package's license is shipped
                       alongside the binary itself and can be found within the
                       `share/licenses/$(src_name)` directory within its prefix.""", "\n")
    # Since this function can be called multiple times, we must make sure that
    # the note is written only once.  Do nothing it is already there.
    if !startswith(license, first(note_lines))
        open(joinpath(code_dir, "LICENSE"), "w") do io
            println.(Ref(io), note_lines)
            println(io)
            println(io, license)
        end
    end
    # We used to have a duplicate license file, remove it.
    rm(joinpath(code_dir, "LICENSE.md"); force=true)

    # Add a Project.toml
    project = build_project_dict(src_name, build_version, dependencies, julia_compat; lazy_artifacts=lazy_artifacts)
    open(joinpath(code_dir, "Project.toml"), "w") do io
        Pkg.TOML.print(io, project)
    end

    # Add a `.gitignore`
    open(joinpath(code_dir, ".gitignore"), "w") do io
        println(io, "override/")
    end
end

function push_jll_package(name, build_version;
                          code_dir = joinpath(Pkg.devdir(), "$(name)_jll"),
                          deploy_repo = "JuliaBinaryWrappers/$(name)_jll.jl",
                          gh_auth = Wizard.github_auth(;allow_anonymous=false),
                          gh_username = gh_get_json(DEFAULT_API, "/user"; auth=gh_auth)["login"])
    # Next, push up the wrapper code repository
    wrapper_repo = LibGit2.GitRepo(code_dir)
    LibGit2.add!(wrapper_repo, ".")
    commit = LibGit2.commit(wrapper_repo, "$(name)_jll build $(build_version)")
    Wizard.with_gitcreds(gh_username, gh_auth.token) do creds
        refspecs = ["refs/heads/main"]
        # Fetch the remote repository, to have the relevant refspecs up to date.
        LibGit2.fetch(
            wrapper_repo;
            refspecs=refspecs,
            credentials=creds,
        )
        LibGit2.branch!(wrapper_repo, "main", string(LibGit2.GitHash(commit)); track="main")
        LibGit2.push(
            wrapper_repo;
            refspecs=refspecs,
            remoteurl="https://github.com/$(deploy_repo).git",
            credentials=creds,
        )
    end
end

# For historical reasons, our UUIDs are generated with some rather strange constants
function bb_specific_uuid5(namespace::UUID, key::String)
    data = [reinterpret(UInt8, [namespace.value]); codeunits(key)]
    u = reinterpret(UInt128, sha1(data)[1:16])[1]
    u &= 0xffffffffffff0fff3fffffffffffffff
    u |= 0x00000000000050008000000000000000
    return UUID(u)
end
const uuid_package = UUID("cfb74b52-ec16-5bb7-a574-95d9e393895e")
# For even more interesting historical reasons, we append an extra
# "_jll" to the name of the new package before computing its UUID.
jll_uuid(name) = bb_specific_uuid5(uuid_package, "$(name)_jll")
function build_project_dict(name, version, dependencies::Array{Dependency}, julia_compat::String=DEFAULT_JULIA_VERSION_SPEC; lazy_artifacts::Bool=false, kwargs...)
    function has_compat_info(d::Dependency)
        r = Pkg.Types.VersionRange()
        return isa(d.pkg.version, VersionNumber) ||
               length(d.pkg.version.ranges) != 1 ||
               d.pkg.version.ranges[1] != r
    end
    function exactly_this_version(v::VersionNumber)
        return string("=", VersionNumber(v.major, v.minor, v.patch))
    end
    function exactly_this_version(v::Pkg.Types.VersionSpec)
        if length(v.ranges) == 1 &&
           v.ranges[1].lower.n == 3 &&
           v.ranges[1].lower == v.ranges[1].upper
           return string("=", v)
       end
       return string(v)
    end
    exactly_this_version(v) = v
    Pkg.Types.semver_spec(julia_compat) # verify julia_compat is valid
    project = Dict(
        "name" => "$(name)_jll",
        "uuid" => string(jll_uuid("$(name)_jll")),
        "version" => string(version),
        "deps" => Dict{String,Any}(),
        # We require at least Julia 1.3+, for Pkg.Artifacts support, but we only claim
        # Julia 1.0+ by default so that empty JLLs can be installed on older versions.
        "compat" => Dict{String,Any}("JLLWrappers" => "1.2.0",
                                     "julia" => "$(julia_compat)")
    )
    for dep in dependencies
        depname = getname(dep)
        project["deps"][depname] = string(jll_uuid(depname))
        if has_compat_info(dep)
            project["compat"][depname] = string(exactly_this_version(dep.pkg.version))
        end
    end
    # Always add Libdl, Pkg and Artifacts as dependencies
    # Once we stop supporting Julia 1.5-, we can drop the `Pkg` requirement.
    stdlibs = isdefined(Pkg.Types, :stdlib) ? Pkg.Types.stdlib : Pkg.Types.stdlibs
    project["deps"]["Libdl"] = "8f399da3-3557-5675-b5ff-fb832c97cbdb"
    project["deps"]["Pkg"] = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
    project["deps"]["Artifacts"] = "56f22d72-fd6d-98f1-02f0-08ddc0907c33"
    project["deps"]["JLLWrappers"] = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210"
    if lazy_artifacts
        project["deps"]["LazyArtifacts"] = "4af54fe1-eca0-43a8-85a7-787d91b784e3"
    end

    return project
end


text:
module TestModule

using Test
using MLStyle

MODULE = TestModule

@use GADT

include("MQuery/test.jl")
include("when.jl")
include("active_patterns.jl")
include("exception.jl")
include("render.jl")
include("pervasive.jl")
include("expr_template.jl")
include("gallery/simple.jl")
include("match.jl")
include("pattern.jl")
include("dot_expression.jl")
include("adt.jl")
include("typelevel.jl")
include("untyped_lam.jl")
include("nothing.jl")

end

ext: jl


text:
using QuasiArrays, Test
import QuasiArrays: MemoryLayout, QuasiArrayLayout, ApplyStyle, QuasiArrayApplyStyle

@testset "Special functions" begin
    @testset "^" begin
        A = QuasiArray(randn(3,3), (0:0.5:1, 0:0.5:1))
        @test MemoryLayout(typeof(A)) isa QuasiArrayLayout
        @test ApplyStyle(*,typeof(A),typeof(A)) isa QuasiArrayApplyStyle
        @test A*A isa QuasiArray{Float64}
        @test A^2 == A^big(2) == A*A
        @test A^4 == A*A*A*A
        @test A^big(2) isa QuasiArray{Float64}
        @test A^(-1) == inv(A) == QuasiArray(inv(A.parent), A.axes)
    end
end


text:
# This file is a part of ParallelProcessingTools.jl, licensed under the MIT License (MIT).

using Test
using ParallelProcessingTools

using Distributed

@testset "onprocs" begin
    @testset "worker-init" begin
        if length(workers()) < 2
            addprocs(2)
        end
        eval(:(@everywhere using Distributed))
        @test length(workers()) >= 2
    end

    @testset "macro onprocs" begin
        @everywhere using ParallelProcessingTools, Base.Threads

        @test (@onprocs workers() myid()) == workers()

        threadinfo = [collect(1:n) for n in [fetch(@spawnat w nthreads()) for w in workers()]]
        ref_result = ((w,t) -> (proc = w, threads = t)).(workers(), threadinfo)

        @test (@onprocs workers() begin
            tl = ThreadLocal(0)
            @onthreads allthreads() tl[] = threadid()
            (proc = myid(), threads = getallvalues(tl))
        end) == ref_result
    end

    @testset "macro mp_async" begin
        @test begin
            n = 128
            A = Vector{Future}(undef, n)
            @sync for i in 1:n
                A[i] = @mp_async begin
                    @assert myid() != 1
                    log(i)
                end
            end
            fetch.(A) == log.(1:n)
        end
    end

    @testset "Examples" begin
        @test begin
            workers() == (@onprocs workers() myid())
        end
    end
end


text: include("planners/fmt.jl")


text:
"""
   RSDSGEModel(file)
Construct a RSDSGE model from a model file. Pass in the path as a string to the file location.
For information on how to write a model file, see the model file documentation, or the examples contained in examples/
Returns your model as an instance of type RSDSGEModel.
"""
function RSDSGEModel(file::String)

    f = open(file)
	modelfilestring = readlines(f)
    close(f)

    # Find the indexes of important things
    ends = Bool[strip(line) == "end" for line in modelfilestring]
    ends = (1:length(modelfilestring))[ends]
    parameters = findline("parameters:",modelfilestring)
    parametervalues = findline("parametervalues:",modelfilestring)
    shocks = findline("shocks:",modelfilestring)
    vars = findline("vars:",modelfilestring)
    equations = findline("equations:",modelfilestring)
    transmatrix = findline("transmatrix: [",modelfilestring)
    transmatrixend = minimum((transmatrix:length(modelfilestring))[Bool[endswith(strip(line),"]") for line in modelfilestring[transmatrix:end]]])
    ssguess = findline("ssguess:",modelfilestring)

    equationend = minimum(ends[ends.>equations])
    ssguessend = minimum(ends[ends.>ssguess])
    valuesend = minimum(ends[ends.>parametervalues])

    vars = parsevars(modelfilestring[vars])
    shocks = parseshocks(modelfilestring[shocks])
    parameters = parseparameters(modelfilestring[parameters])
    parametervalues = parseparametervalues(modelfilestring[(parametervalues+1):(valuesend-1)],parameters)
    equations = parseequations(modelfilestring[(equations+1):(equationend-1)])
    transmatrix = parsetransmatrix(modelfilestring[transmatrix:transmatrixend])
    ssguess_dict = splitonequals(modelfilestring[(ssguess+1):(ssguessend-1)])

    ssguess = zeros(Float64,length(vars)) # Default to zero if unset
    for (key,val) in ssguess_dict
	if countnz(vars.==key) == 0
	    error("You have set a steady state guess for $key, but it is not a declared variable.")
        else
            ssguess[vars.==key] = val
        end
    end

    return RSDSGEModel(vars,shocks,parameters,equations,parametervalues,transmatrix,ssguess)
end

function findline(keyword,modelfilestring)
    indices = Bool[startswith(strip(line),keyword) for line in modelfilestring]
    if countnz(indices) != 1
	error("Model file contains $(countnz(indices)) occurences of the $keyword keyword. It must appear only once.")
    end

    return (1:length(modelfilestring))[indices][1] # The extra one gets the scalar, rather than a 1 element array
end

function parsevars(vars)
    vars = strip(vars[6:end]) # Chop off the keyword and whitespace
    return splitoncomma(vars)
end

function parseshocks(shocks)
    shocks = strip(shocks[8:end]) # Chop off the keyword and whitespace
    return splitoncomma(shocks)
end

function parseparameters(parameters)
    parameters = strip(parameters[12:end]) # Chop off the keyword and whitespace
    return splitoncomma(parameters)
end

function parsetransmatrix(transmatrixstring)
    numregimes = length(transmatrixstring)
    transmatrix = Array{Float64}(numregimes,numregimes)
    for r in 1:numregimes
	if r == 1
	    # First line, need to strip off the first bit
	    warn("Make this more robust by finding [ instead of assuming it is 15")
	    transmatrixstring[r] = transmatrixstring[r][15:end]
	elseif r == numregimes
	    # Last line, strip off the closing ]
	    transmatrixstring[r] = strip(transmatrixstring[r])
	    transmatrixstring[r] = transmatrixstring[r][1:(end-1)]
	end
	vals = split(strip(transmatrixstring[r]), " ")
	for (rp,val) in enumerate(vals)
	    transmatrix[r,rp] = parse(Float64,val)
	end
    end
    return transmatrix
end

function parsetuple(input)
    if input[1] == '(' && input[end] == ')'
	input = input[2:(end-1)]
	vals = split(input,",")
	out = ()
	for val in vals
	    out = (out...,parse(Float64,val))
	end
	return out
    else
	println(input[1])
	println(input[end])
	error("$input is not valid tuple syntax.")
    end
end

function splitonequals(input)
    out = Dict{String,Union{Float64,Tuple}}()
    for i in eachindex(input)
	pair = split(input[i],"=")
	if length(pair)!=2
	    error("Error reading line '$(in[i])'; wrong number of equals signs.")
        else
	    if contains(pair[2],"(")
		out[strip(pair[1])] = parsetuple(strip(pair[2]))
	    else
		out[strip(pair[1])] = parse(Float64,strip(pair[2]))
	    end
        end
    end
    return out
end

function parseparametervalues(values,parameters)
    parametervalues = Array{Any}(length(parameters))
    values_dict = splitonequals(values)
    unset = trues(length(parameters))
    for (key,val) in values_dict
	if countnz(parameters.==key) == 0
	    error("You have set a parameter value for $key, but it is not a declared parameter.")
        else
            parametervalues[parameters.==key] = val
	    unset[parameters.==key] = false
        end
    end
    if any(unset)
	error("You have not set a value for parameter(s): $(parameters[unset]).")
    end
    return parametervalues
end

function parseequations(equations)
    out = Array{String}(length(equations))
    for i in eachindex(equations)
	out[i] = strip(equations[i])
    end
    return out
end

function splitoncomma(in)
    list = split(in,",")
    out = Array{String}(length(list))
    for i in eachindex(list)
	out[i] = strip(list[i])
    end
    return out
end


text:
"""
        getcomposites(file2d,file3d,diagnostics_dir,exp_,initial_timeindex,final_timeindex,partnumber,outputInterval)

"""
function get_composites_nosmoothing(file2d,file3d,exp_name)

    @info "Starting composite routine"
    diagnostics_data_dir = "/global/cscratch1/sd/aramreye/for_postprocessing/ApeBudgetOutputs_nosmoothing/"
    composites_data_dir = "/global/cscratch1/sd/aramreye/for_postprocessing/CompositeOutputs_nosmoothing/"
    azimuthal_averages_data_dir = "/global/cscratch1/sd/aramreye/for_postprocessing/AzimuthualAverageOutputs_nosmoothing/"

    variables_diag = ["convec_heating_anomaly","rad_heating_anomaly","buoyancy_anomaly"]
    variables_orig_3d = ["U","V","QV","TABS","QRAD","PP","W"]
    variables_orig_2d = ["USFC","VSFC","PW","Prec","LHF","SHF"]

    initial_timeindex = 601
    #    initial_timeindex = 1195
    final_timeindex = 1200

    iterator_time_2d    = initial_timeindex*2-1:2:final_timeindex*2
    iterator_time_3d    = initial_timeindex:1:final_timeindex

    radiusbins = 1000:2000:512000;
    ds3d                = Dataset(file3d)
    ds2d                = Dataset(file2d)
    z                   = variable(ds3d,"z")[:]                       :: Array{Float32,1}
    t                   = variable(ds3d,"time")[iterator_time_3d]     :: Array{Float32,1}
    surface_pressure    = variable(ds2d,"PSFC")[:,:,iterator_time_2d] :: Array{Float32,3}
    pressure_anomaly    = surface_pressure .- mean(surface_pressure,dims=(1,2))

    ##Create buffers to reuse on allocating functions

    size_var_of_interest_2d = size(surface_pressure)[1:end-1]
    size_var_of_interest_3d = (size_var_of_interest_2d...,length(z))
    ## Create 3D buffers
    #var_of_interest_3d      = zeros(size_var_of_interest_3d)
    composite_addition_3d   = zeros(Float32,size_var_of_interest_3d)
    buf_3d_2                = similar(composite_addition_3d)
    buf_3d_1                = similar(composite_addition_3d)
    azimuthal_average_3d    = zeros(Float32,length(radiusbins) - 1,length(z))
    ## Create 2D buffers
    #var_of_interest_2d      = zeros(size_var_of_interest_2d)
    composite_addition_2d   = zeros(Float32,size_var_of_interest_2d)
    buf_2d_1                = similar(composite_addition_2d)
    buf_2d_2                = similar(composite_addition_2d)
    azimuthal_average_2d    = zeros(Float32,length(radiusbins) - 1)
    @info size_var_of_interest_2d size_var_of_interest_3d

    centers_labels_and_cyclones = @views [
        detect_cyclones!(buf_2d_1, buf_2d_2, pressure_anomaly[:,:,timeindex],-5,2000) for
        timeindex in 1:size(surface_pressure,3)]; pressure_anomaly = [];

    composite_filename_withmask = string(composites_data_dir,exp_name        ,"_withmask.jld")
    composite_filename_nomask   = string(composites_data_dir,exp_name        ,"_nomask.jld")
    azimuthal_filename_withmask = string(azimuthal_averages_data_dir,exp_name,"_withmask.jld")
    azimuthal_filename_nomask   = string(azimuthal_averages_data_dir,exp_name,"_nomask.jld")

    ### *** add allcyclones ***
    @info "Creating composite for surface pressure "
    flush(stdout)
    composite_addition_2d .= 0.0
    buf_2d_1              .= 0.0
    buf_2d_2              .= 0.0
    azimuthal_average_2d  .= 0.0
    composite_and_radialaverage!(azimuthal_average_2d,composite_addition_2d,buf_2d_1,
    buf_2d_2,centers_labels_and_cyclones,radiusbins,surface_pressure;maskcyclones = true)

    @info "Writing"
    flush(stdout)
    h5open(composite_filename_withmask, "w") do file
        write(file,"surface_pressure",composite_addition_2d)
    end

    h5open(azimuthal_filename_withmask, "w") do file
        write(file,"surface_pressure",azimuthal_average_2d)
    end
    GC.gc()
    GC.gc()
    ## Create products without mask
    @info "Creating composite without mask for: surface_pressure"
    flush(stdout)
    composite_addition_2d .= 0.0
    buf_2d_1              .= 0.0
    buf_2d_2              .= 0.0
    azimuthal_average_2d  .= 0.0
    composite_and_radialaverage!(azimuthal_average_2d,composite_addition_2d,buf_2d_1,buf_2d_2
    ,centers_labels_and_cyclones,radiusbins,surface_pressure;maskcyclones = false)
    @info "Writing"
    flush(stdout)
    h5open(composite_filename_nomask, "w") do file
        write(file,"surface_pressure",composite_addition_2d)
    end
    h5open(azimuthal_filename_nomask, "w") do file
        write(file,"surface_pressure",azimuthal_average_2d)
    end
    GC.gc()
    GC.gc()

    for current_variable in variables_orig_3d
        ##NCDatasets.load!(ds["PSFC"].var,a,:,:,:)

        var_of_interest = variable(ds3d,current_variable)[:,:,:,iterator_time_3d]    :: Array{Float32,4}
        composite_addition_3d .= 0.0
        buf_3d_1              .= 0.0
        buf_3d_2              .= 0.0
        azimuthal_average_3d  .= 0.0

        ## Create products with mask
        @info "Creating composite for: " current_variable
        flush(stdout)
        composite_and_radialaverage!(azimuthal_average_3d,composite_addition_3d,buf_3d_1,buf_3d_2,centers_labels_and_cyclones,radiusbins,var_of_interest;maskcyclones = true)
        @info "Writing"
        flush(stdout)
        h5open(composite_filename_withmask, "r+",mmaparrays=true) do file
            write(file,current_variable,composite_addition_3d)
        end
        h5open(azimuthal_filename_withmask, "r+",mmaparrays=true) do file
            write(file,current_variable,azimuthal_average_3d)
        end
        GC.gc()
        GC.gc()
        ## Create products without mask
        @info "Creating composite without mask for: " current_variable
        flush(stdout)
        composite_addition_3d .= 0.0
        buf_3d_1              .= 0.0
        buf_3d_2              .= 0.0
        azimuthal_average_3d  .= 0.0
        composite_and_radialaverage!(azimuthal_average_3d,composite_addition_3d,buf_3d_1,buf_3d_2,centers_labels_and_cyclones,radiusbins,var_of_interest;maskcyclones = false)
        @info "Writing"
        flush(stdout)
        h5open(composite_filename_nomask, "r+",mmaparrays=true) do file
            write(file,current_variable,composite_addition_3d)
        end
        h5open(azimuthal_filename_nomask, "r+",mmaparrays=true) do file
            write(file,current_variable,azimuthal_average_3d)
        end
        GC.gc()
    end

    for current_variable in variables_orig_2d
        composite_addition_2d .= 0.0
        buf_2d_1              .= 0.0
        buf_2d_2              .= 0.0
        azimuthal_average_2d  .= 0.0

        var_of_interest = variable(ds2d,current_variable)[:,:,iterator_time_2d]    :: Array{Float32,3}

        ## Create products with mask
        @info "Creating composite for: " current_variable
        flush(stdout)
        composite_and_radialaverage!(azimuthal_average_2d,composite_addition_2d,buf_2d_1,buf_2d_2,centers_labels_and_cyclones,radiusbins,var_of_interest;maskcyclones = true)
        @info "Writing"
        flush(stdout)
        h5open(composite_filename_withmask, "r+",mmaparrays=true) do file
            write(file,current_variable,composite_addition_2d)
        end
        h5open(azimuthal_filename_withmask, "r+",mmaparrays=true) do file
            write(file,current_variable,azimuthal_average_2d)
        end
        GC.gc()
        GC.gc()
        ## Create products without mask
        @info "Creating composite without mask for: " current_variable
        flush(stdout)
        composite_addition_2d .= 0.0
        buf_2d_1              .= 0.0
        buf_2d_2              .= 0.0
        azimuthal_average_2d  .= 0.0
        composite_and_radialaverage!(azimuthal_average_2d,composite_addition_2d,buf_2d_1,buf_2d_2,centers_labels_and_cyclones,radiusbins,var_of_interest;maskcyclones = false)
        @info "Writing"
        flush(stdout)
        h5open(composite_filename_nomask, "r+",mmaparrays=true) do file
            write(file,current_variable,composite_addition_2d)
        end
        h5open(azimuthal_filename_nomask, "r+",mmaparrays=true) do file
            write(file,current_variable,azimuthal_average_2d)
        end
        GC.gc()
    end

    for current_variable in variables_diag
        var_of_interest = gather_diagnostic_3d(diagnostics_data_dir,exp_name,current_variable)   :: Array{Float32,4}
        composite_addition_3d .= 0.0
        buf_3d_1              .= 0.0
        buf_3d_2              .= 0.0
        azimuthal_average_3d  .= 0.0
        ## Create products with mask
        @info "Creating composite for: " current_variable
        flush(stdout)
        composite_and_radialaverage!(azimuthal_average_3d,composite_addition_3d,buf_3d_1,buf_3d_2,centers_labels_and_cyclones,radiusbins,var_of_interest;maskcyclones = true)
        @info "Writing"
        flush(stdout)
        h5open(composite_filename_withmask, "r+",mmaparrays=true) do file
            write(file,current_variable,composite_addition_3d)
        end
        h5open(azimuthal_filename_withmask, "r+",mmaparrays=true) do file
            write(file,current_variable,azimuthal_average_3d)
        end
        GC.gc()
        GC.gc()
        ## Create products without mask
        @info "Creating composite without mask for: " current_variable
        flush(stdout)
        composite_addition_3d .= 0.0
        buf_3d_1              .= 0.0
        buf_3d_2              .= 0.0
        azimuthal_average_3d  .= 0.0
        composite_and_radialaverage!(azimuthal_average_3d,composite_addition_3d,buf_3d_1,buf_3d_2,centers_labels_and_cyclones,radiusbins,var_of_interest;maskcyclones = false)
        @info "Writing"
        flush(stdout)
        h5open(composite_filename_nomask, "r+",mmaparrays=true) do file
            write(file,current_variable,composite_addition_3d)
        end
        h5open(azimuthal_filename_nomask, "r+",mmaparrays=true) do file
            write(file,current_variable,azimuthal_average_3d)
        end
        GC.gc()
        GC.gc()
    end

    close(ds3d)
    close(ds2d)

    return nothing

end

"""
        getcomposites(file2d,file3d,diagnostics_dir,exp_,initial_timeindex,final_timeindex,partnumber,outputInterval)

"""
function cyclone_detect_and_save(file2d,file3d,exp_name)

    @info "Starting composite routine"
    diagnostics_data_dir = "/global/cscratch1/sd/aramreye/for_postprocessing/ApeBudgetOutputs_nosmoothing/"
    composites_data_dir = "/global/cscratch1/sd/aramreye/for_postprocessing/CompositeOutputs_nosmoothing/"
    azimuthal_averages_data_dir = "/global/cscratch1/sd/aramreye/for_postprocessing/AzimuthualAverageOutputs_nosmoothing/"

    variables_diag = ["convec_heating_anomaly","rad_heating_anomaly","buoyancy_anomaly"]
    variables_orig_3d = ["U","V","QV","TABS","QRAD","PP","W"]
    variables_orig_2d = ["USFC","VSFC","PW","Prec","LHF","SHF"]

    initial_timeindex = 601
    #    initial_timeindex = 1195
    final_timeindex = 620
#    final_timeindex = 1200

    iterator_time_2d    = initial_timeindex*2-1:2:final_timeindex*2
    iterator_time_3d    = initial_timeindex:1:final_timeindex

    radiusbins = 1000:2000:512000;
    ds3d                = Dataset(file3d)
    ds2d                = Dataset(file2d)
    z                   = variable(ds3d,"z")[:]                       :: Array{Float32,1}
    t                   = variable(ds3d,"time")[iterator_time_3d]     :: Array{Float32,1}
    surface_pressure    = variable(ds2d,"PSFC")[:,:,iterator_time_2d] :: Array{Float32,3}
    pressure_anomaly    = surface_pressure .- mean(surface_pressure,dims=(1,2))

    ##Create buffers to reuse on allocating functions

    size_var_of_interest_2d = size(surface_pressure)[1:end-1]
    size_var_of_interest_3d = (size_var_of_interest_2d...,length(z))
    ## Create 3D buffers
    #var_of_interest_3d      = zeros(size_var_of_interest_3d)
    composite_addition_3d   = zeros(Float32,size_var_of_interest_3d)
    buf_3d_2                = similar(composite_addition_3d)
    buf_3d_1                = similar(composite_addition_3d)
    azimuthal_average_3d    = zeros(Float32,length(radiusbins) - 1,length(z))
    ## Create 2D buffers
    #var_of_interest_2d      = zeros(size_var_of_interest_2d)
    composite_addition_2d   = zeros(Float32,size_var_of_interest_2d)
    buf_2d_1                = similar(composite_addition_2d)
    buf_2d_2                = similar(composite_addition_2d)
    azimuthal_average_2d    = zeros(Float32,length(radiusbins) - 1)
    @info size_var_of_interest_2d size_var_of_interest_3d

    for timeindex in 1:size(surface_pressure,3)
        centers_labels_and_cyclones = @views detect_cyclones!(buf_2d_1, buf_2d_2, pressure_anomaly[:,:,timeindex],-5,2000)
        @info "Data types " typeof(centers_labels_and_cyclones[1]) centers_labels_and_cyclones[1]  typeof(centers_labels_and_cyclones[2])
    end
        pressure_anomaly = [];

#    jldopen(string(composites_data_dir,"centers_detection_",exp_name,".jld"), "w") do file
#        addrequire(file, AvailablePotentialEnergyFramework)
#        write(file, "centers_data", centers_labels_and_cyclones)  # alternatively, say "@write file A"
#    end

    composite_filename_withmask = string(composites_data_dir,exp_name        ,"_withmask.jld")
    composite_filename_nomask   = string(composites_data_dir,exp_name        ,"_nomask.jld")
    azimuthal_filename_withmask = string(azimuthal_averages_data_dir,exp_name,"_withmask.jld")
    azimuthal_filename_nomask   = string(azimuthal_averages_data_dir,exp_name,"_nomask.jld")

    close(ds3d)
    close(ds2d)

    return nothing

end


text:
include("../imports.jl")

@testset ExtendedTestSet "`Scheduler`" begin
    nbatches = 16
    schedule = Schedule([0, 2], [1e-4, 1e-6], Animations.linear())
    learner = testlearner(coeff = 3, Recorder(), Scheduler(LearningRate => schedule), nbatches = nbatches)

    epoch!(learner, TrainingPhase())
    @test isapprox(last(learner.cbstate.hyperparams[:LearningRate])[2], ((1e-4 + 1e-6) / 2), atol = 0.1)

    epoch!(learner, TrainingPhase())
    @test isapprox(last(learner.cbstate.hyperparams[:LearningRate])[2], 1e-6, atol =0.1)
end


text:
export alloc_special

"Allocates on-device memory statically from the specified address space."
@generated function alloc_special(::Val{id}, ::Type{T}, ::Val{as}, ::Val{len}=Val(0)) where {id,T,as,len}
    eltyp = convert(LLVMType, T)

    # old versions of GPUArrays invoke _shmem with an integer id; make sure those are unique
    if !isa(id, String) || !isa(id, Symbol)
        id = "alloc_special_$id"
    end

    T_ptr = convert(LLVMType, DevicePtr{T,as})

    # create a function
    llvm_f, _ = create_function(T_ptr)

    # create the global variable
    mod = LLVM.parent(llvm_f)
    gv_typ = LLVM.ArrayType(eltyp, len)
    gv = GlobalVariable(mod, gv_typ, string(id), convert(Int, as))
    if len > 0
        linkage!(gv, LLVM.API.LLVMInternalLinkage)
        #initializer!(gv, null(gv_typ))
    end

    # by requesting a larger-than-datatype alignment, we might be able to vectorize.
    # TODO: Make the alignment configurable
    alignment!(gv, Base.max(32, Base.datatype_alignment(T)))

    # generate IR
    Builder(JuliaContext()) do builder
        entry = BasicBlock(llvm_f, "entry", JuliaContext())
        position!(builder, entry)

        ptr_with_as = gep!(builder, gv, [ConstantInt(0, JuliaContext()),
                                         ConstantInt(0, JuliaContext())])

        val = ptrtoint!(builder, ptr_with_as, T_ptr)
        ret!(builder, val)
    end

    call_function(llvm_f, DevicePtr{T,as})
end


text:
#  Copyright 2017, Iain Dunning, Joey Huchette, Miles Lubin, and contributors
#  This Source Code Form is subject to the terms of the Mozilla Public
#  License, v. 2.0. If a copy of the MPL was not distributed with this
#  file, You can obtain one at https://mozilla.org/MPL/2.0/.

"""
    struct SparseAxisArray{T,N,K<:NTuple{N, Any}} <: AbstractArray{T,N}
        data::Dict{K,T}
    end

`N`-dimensional array with elements of type `T` where only a subset of the
entries are defined. The entries with indices `idx = (i1, i2, ..., iN)` in
`keys(data)` has value `data[idx]`. Note that as opposed to
`SparseArrays.AbstractSparseArray`, the missing entries are not assumed to be
`zero(T)`, they are simply not part of the array. This means that the result of
`map(f, sa::SparseAxisArray)` or `f.(sa::SparseAxisArray)` has the same sparsity
structure than `sa` even if `f(zero(T))` is not zero.

# Example
```jldoctest; setup=:(using JuMP)
julia> dict = Dict((:a, 2) => 1.0, (:a, 3) => 2.0, (:b, 3) => 3.0)
Dict{Tuple{Symbol,Int64},Float64} with 3 entries:
  (:b, 3) => 3.0
  (:a, 2) => 1.0
  (:a, 3) => 2.0

julia> array = JuMP.Containers.SparseAxisArray(dict)
JuMP.Containers.SparseAxisArray{Float64,2,Tuple{Symbol,Int64}} with 3 entries:
  [b, 3]  =  3.0
  [a, 2]  =  1.0
  [a, 3]  =  2.0

julia> array[:b, 3]
3.0
```
"""
struct SparseAxisArray{T,N,K<:NTuple{N,Any}} <: AbstractArray{T,N}
    data::Dict{K,T}
end

Base.length(sa::SparseAxisArray) = length(sa.data)
Base.IteratorSize(::Type{<:SparseAxisArray}) = Base.HasLength()
# By default `IteratorSize` for `Generator{<:AbstractArray{T,N}}` is
# `HasShape{N}`
Base.IteratorSize(::Type{Base.Generator{<:SparseAxisArray}}) = Base.HasLength()
Base.iterate(sa::SparseAxisArray, args...) = iterate(values(sa.data), args...)

# A `length` argument can be given because `IteratorSize` is `HasLength`
function Base.similar(
    ::SparseAxisArray{S,N,K},
    ::Type{T},
    length::Integer = 0,
) where {S,T,N,K}
    d = Dict{K,T}()
    if !iszero(length)
        sizehint!(d, length)
    end
    return SparseAxisArray(d)
end

function Base.mapreduce(f, op, sa::SparseAxisArray)
    return mapreduce(f, op, values(sa.data))
end
Base.:(==)(sa1::SparseAxisArray, sa2::SparseAxisArray) = sa1.data == sa2.data

############
# Indexing #
############

Base.haskey(sa::SparseAxisArray, idx) = haskey(sa.data, idx)
function Base.haskey(sa::SparseAxisArray{T,1,Tuple{I}}, idx::I) where {T,I}
    return haskey(sa.data, (idx,))
end

# Error for sa[..., :, ...]
function _colon_error() end
function _colon_error(::Colon, args...)
    return throw(
        ArgumentError(
            "Indexing with `:` is not supported by" *
            " Containers.SparseAxisArray",
        ),
    )
end
_colon_error(arg, args...) = _colon_error(args...)

function Base.setindex!(
    d::SparseAxisArray{T,N,K},
    value,
    idx::K,
) where {T,N,K<:NTuple{N,Any}}
    return setindex!(d, value, idx...)
end
function Base.setindex!(d::SparseAxisArray{T,N}, value, idx...) where {T,N}
    length(idx) < N && throw(BoundsError(d, idx))
    _colon_error(idx...)
    return setindex!(d.data, value, idx)
end

function Base.getindex(
    d::SparseAxisArray{T,N,K},
    idx::K,
) where {T,N,K<:NTuple{N,Any}}
    return getindex(d, idx...)
end
function Base.getindex(d::SparseAxisArray{T,N}, idx...) where {T,N}
    length(idx) < N && throw(BoundsError(d, idx))
    _colon_error(idx...)
    return getindex(d.data, idx)
end

Base.eachindex(d::SparseAxisArray) = keys(d.data)

# Need to define it as indices may be non-integers
Base.to_index(d::SparseAxisArray, idx) = idx

Base.IndexStyle(::Type{<:SparseAxisArray}) = IndexAnyCartesian()
# eachindex redirect to keys
Base.keys(::IndexAnyCartesian, d::SparseAxisArray) = keys(d)

################
# Broadcasting #
################

# `broadcast(f, args...)` starts by creating an object:
#
#     bc = Broadcast.broadcasted(f, args...)
#
# If one of the arguments is `SparseAxisArray`, we want `bc` to be of type
# `Broadcasted{BroadcastStyle)`. Then, the following functions is called:
# `copy(bc)`. It first attempts to determine the return type `ElType` of `f` on
# applied on elements of `args`. If `ElType` is a concrete type, it returns:
#
#     copyto!(similar(bc, ElType), bc)
#
# Otherwise, it calls `extrude` on each `args`, maps the first element, set
# `ElType` to its type and calls `copyto_nonleaf!`.

Base.Broadcast.extrude(sa::SparseAxisArray) = sa

# Need to define it as indices may be non-integers
Base.Broadcast.newindex(d::SparseAxisArray, idx) = idx

struct BroadcastStyle{N,K} <: Broadcast.BroadcastStyle end
function Base.BroadcastStyle(::BroadcastStyle, ::Base.BroadcastStyle)
    return throw(
        ArgumentError(
            "Cannot broadcast Containers.SparseAxisArray with" *
            " another array of different type",
        ),
    )
end
# Scalars can be used with SparseAxisArray in broadcast
function Base.BroadcastStyle(
    ::BroadcastStyle{N,K},
    ::Base.Broadcast.DefaultArrayStyle{0},
) where {N,K}
    return BroadcastStyle{N,K}()
end
function Base.BroadcastStyle(::Type{<:SparseAxisArray{T,N,K}}) where {T,N,K}
    return BroadcastStyle{N,K}()
end

function Base.similar(
    b::Base.Broadcast.Broadcasted{BroadcastStyle{N,K}},
    ::Type{T},
) where {T,N,K}
    return SparseAxisArray(Dict{K,T}())
end

# Check that all `SparseAxisArray`s involved have the same indices. The other
# arguments are scalars
function check_same_eachindex(each_index) end
function check_same_eachindex(each_index, not_sa, args...)
    return check_same_eachindex(eachindex, args...)
end
function check_same_eachindex(each_index, sa::SparseAxisArray, args...)
    if Set(each_index) != Set(eachindex(sa))
        throw(
            ArgumentError(
                "Cannot broadcast Containers.SparseAxisArray with" *
                " different indices",
            ),
        )
    end
    return check_same_eachindex(eachindex, args...)
end
_eachindex(not_sa, args...) = _eachindex(args...)
function _eachindex(sa::SparseAxisArray, args...)
    each_index = eachindex(sa)
    check_same_eachindex(each_index, args...)
    return each_index
end
# Need to define it as it falls back to `axes` by default
function Base.eachindex(bc::Base.Broadcast.Broadcasted{<:BroadcastStyle})
    return _eachindex(bc.args...)
end

# The fallback uses `axes` but recommend in the docstring to create a custom
# method for custom style if needed.
function Base.Broadcast.instantiate(
    bc::Base.Broadcast.Broadcasted{<:BroadcastStyle},
)
    return bc
end

# The generic method in `Base` is `getindex(::Broadcasted, ::Union{Integer, CartesianIndex})`
# which is not applicable here since the index is not integer
# TODO make a change in `Base` so that we don't have to call a function starting
# with an `_`.
function Base.getindex(bc::Base.Broadcast.Broadcasted{<:BroadcastStyle}, I)
    return Base.Broadcast._broadcast_getindex(bc, I)
end

# The generic implementation fall back to converting `bc` to
# `Broadcasted{Nothing}`. It is advised in `Base` to define a custom method for
# custom styles. The fallback for `Broadcasted{Nothing}` is not appropriate as
# indices are not integers for `SparseAxisArray`.
function Base.copyto!(
    dest::SparseAxisArray{T,N,K},
    bc::Base.Broadcast.Broadcasted{BroadcastStyle{N,K}},
) where {T,N,K}
    for key in eachindex(bc)
        dest[key] = bc[key]
    end
    return dest
end

@static if VERSION >= v"1.3"
    # `broadcast_preserving_zero_d` calls `axes(A)` which calls `size(A)` which
    # is not defined. When at least one argument is a `SparseAxisArray`, we can
    # simply redirect `broadcast_preserving_zero_d` to `broadcast` since we know
    # the result won't be zero dimensional.

    # Called by `A * 2`
    function Base.Broadcast.broadcast_preserving_zero_d(
        f,
        A::SparseAxisArray,
        As...,
    )
        return broadcast(f, A, As...)
    end
    # Called by `2 * A`
    function Base.Broadcast.broadcast_preserving_zero_d(
        f,
        x,
        A::SparseAxisArray,
        As...,
    )
        return broadcast(f, x, A, As...)
    end
end

########
# Show #
########

# Inspired from Julia SparseArrays stdlib package
# `Base.summary` is also called from `showerror` on `BoundsError`.
function Base.summary(io::IO, sa::SparseAxisArray)
    num_entries = length(sa.data)
    return print(
        io,
        typeof(sa),
        " with ",
        num_entries,
        isone(num_entries) ? " entry" : " entries",
    )
end
function Base.show(io::IO, ::MIME"text/plain", sa::SparseAxisArray)
    summary(io, sa)
    if !iszero(length(sa.data))
        println(io, ":")
        show(io, sa)
    end
end
Base.show(io::IO, x::SparseAxisArray) = show(convert(IOContext, io), x)
function Base.show(io::IOContext, x::SparseAxisArray)
    # TODO: make this a one-line form
    if isempty(x)
        return show(io, MIME("text/plain"), x)
    end
    limit::Bool = get(io, :limit, false)
    half_screen_rows = limit ? div(displaysize(io)[1] - 8, 2) : typemax(Int)
    key_string(key::Tuple) = join(key, ", ")
    print_entry(i) = i < half_screen_rows || i > length(x) - half_screen_rows
    pad = maximum(
        Int[
            print_entry(i) ? length(key_string(key)) : 0 for
            (i, key) in enumerate(keys(x.data))
        ],
    )
    if !haskey(io, :compact)
        io = IOContext(io, :compact => true)
    end
    for (i, (key, value)) in enumerate(x.data)
        if print_entry(i)
            print(io, "  ", '[', rpad(key_string(key), pad), "]  =  ", value)
            if i != length(x)
                println(io)
            end
        elseif i == half_screen_rows
            println(io, "   ", " "^pad, "   \u22ee")
        end
    end
end

max_stars_repo_name: ethansaxenian/RosettaDecode

max_issues_repo_name: ethansaxenian/RosettaDecode

max_forks_repo_name: ethansaxenian/RosettaDecode


text:
using IntervalArithmetic

using PyCall
using PyPlot

@pyimport matplotlib.patches as patches
@pyimport matplotlib.collections as collections

const rectangle = patches.Rectangle

function make_rectangle(x, y, xwidth, ywidth, color="grey", alpha=0.5)
    patches.Rectangle(
        (x, y), xwidth, ywidth, facecolor=color, alpha=alpha
    )
end

import PyPlot.draw
function draw(box_list::Vector{T}, color="grey", alpha=0.5) where T<:IntervalBox
    patch_list = []
    for box in box_list
        x, y = box
        push!(patch_list,
            make_rectangle(x.lo, y.lo, x.hi-x.lo, y.hi-y.lo, color, alpha))
    end

    ax = gca()
    ax[:add_collection](collections.PatchCollection(patch_list, color=color, alpha=alpha,
    edgecolor="black"))
end


text:
module CESMReader2

    using NCDatasets
    using Formatting
    export FileHandler, getData

    include("nanop.jl")

    function iterFilenames(
        f               :: Function,
        filename_format :: String,
        year_rng        :: Union{Tuple, Array},
    )

        beg_y = year_rng[1]
        end_y = year_rng[2]

        beg_m = 1
        end_m = 12

        beg_t = (beg_y - 1) * 12 + beg_m - 1
        end_t = (end_y - 1) * 12 + end_m - 1

        months = end_t - beg_t + 1

        if months <= 0
            throw(ErrorException("End time should be larger than begin time"))
        end

        if ! ( ( 1 <= beg_m <= 12 ) && ( 1 <= end_m <= 12 ) )
            throw(ErrorException("Invalid month"))
        end

        for y in beg_y:end_y, m in beg_m:end_m

            current_t = (y-1) * 12 + (m-1) - beg_t + 1
            filename = format(filename_format, y, m)
            if_continue = f(filename, y, m)

            if if_continue == false
                println("Break the iteration.")
                break
            end
        end

    end
end

max_stars_repo_name: meteorologytoday/CESM-diagnostic

max_issues_repo_name: meteorologytoday/CESM-diagnostic

max_forks_repo_name: meteorologytoday/CESM-diagnostic


text:
# ------------------------------------------------------------------
# Licensed under the MIT License. See LICENSE in the project root.
# ------------------------------------------------------------------

"""
    SamplingMethod

A method for sampling from geometries.
"""
abstract type SamplingMethod end

"""
    sample(geometry, method)

Sample elements from `geometry` with `method`.
"""
function sample end

# ----------------
# IMPLEMENTATIONS
# ----------------

include("sampling/regular.jl")
include("sampling/uniform.jl")
include("sampling/weighted.jl")
include("sampling/ball.jl")