text:
# using Distributed:nprocs, addprocs
# nprocs()
# addprocs(3)

module Synaprune

    using Flux
    using Flux:softmax, sigmoid
    using Statistics:mean
    using Distributed:addprocs, nprocs, @distributed

    abstract type Layers end

    mutable struct Build <: Layers
        I
        W::Array
        O
        f
    end

    mutable struct Last <: Layers
        I
        W::Array
        O
        f
    end

    struct Freeze
        layer
    end

    function linear(x)
        return x
    end

    function sigmoid(x)
        return Flux.sigmoid.(x)
    end

    Devarg(x) = x[argmax(abs.(x), dims=1)]

    Build(in::Integer, out::Integer, f) = Build(Nothing, 2*rand(out, in) .- 1, Nothing, f)
    Build(in::Integer, out::Integer) = Build(Nothing, 2*rand(out, in) .- 1, Nothing, linear)
    Last(in::Integer, out::Integer, f) = Last(Nothing, 2*rand(out, in) .- 1, Nothing, f)
    Last(in::Integer, out::Integer) = Last(Nothing, 2*rand(out, in) .- 1, Nothing, linear)

    function (m::Layers)(x)
        m.O = m.f(m.W*x)
        m.I = x
        return m.O
    end

    function (f::Freeze)(x)
        return f.layer(x)
    end

    function Update!(f::Freeze, loss, loss1, learning_rate)
        return nothing
    end

    function Standardize(x)
        return x ./ sum(abs.(x), dims=1)
    end

    function Update!(b::Build, loss, loss1, learning_rate)
        Local = Standardize(b.O)
        Xn = Standardize(b.I)
        O, I = size(b.W)
        w = zeros(O, I, size(b.I)[2])
        for i in 1:size(b.I)[2]
            w[:,:,i] = matmul(loss1[i], Local[:,i], Xn[:,i])
        end
        b.W += learning_rate * w[argmax(abs.(w), dims=3)][:,:]
    end

    matmul(loss, Xn) = loss * transpose(Xn)
    matmul(loss1, Local, Xn) = loss1 * (Local * transpose(Xn))

    function Update!(b::Last, loss, loss1, learning_rate)
        Xn = Standardize(b.I)
        O, I = size(L.W)
        w = zeros(O, I, size(L.I)[2])
        for i in 1:size(L.I)[2]
            w[:,:,i] = matmul(loss[:,i], Xn[:,i])
        end
        b.W += learning_rate * w[argmax(abs.(w), dims=3)][:,:]
    end

    function train!(model::Flux.Chain, data, actual, learning_rate)
        pred = model(data)
        error = actual - pred
        error1 = Devarg(error)
        @distributed for m in 1:length(model)
            Update!(model[m], error, error1, learning_rate)
        end
        @show mean(abs.(error))
    end

end

max_stars_repo_name: Jigusheng/SynaPrune

max_issues_repo_name: Jigusheng/SynaPrune

max_forks_repo_name: Jigusheng/SynaPrune


text:
module NeXLSpectrum

using ThreadsX: maximum
using NeXLCore: isinside
using Base: UInt16, UInt64
using Reexport
using Requires

using Dates
using Images
using ImageAxes: AxisArrays, AxisArray, axes, axisnames, axisvalues
using Unitful: mm
using FileIO
using Mmap
using EzXML: readxml
using Polynomials: ImmutablePolynomial, fit, printpoly, roots, derivative, coeffs
using LinearAlgebra
using LoopVectorization
using Statistics
using DataAPI
using CSV
using DataFrames
using Interpolations: LinearInterpolation, AbstractInterpolation, bounds
using LsqFit: curve_fit
using ThreadsX

@reexport using NeXLCore
@reexport using NeXLMatrixCorrection

include("features.jl")
include("detector.jl")
export EnergyScale # Abstract: Detector energy calibration function
export LinearEnergyScale # A linear detector energy calibration function
export PolyEnergyScale # A polynomial energy calibration function
export Resolution # Abstract: Detector resolution function
export MnKaResolution # Fiori function resolution function
export Detector # Abstract: X-ray detector
export EscapeArtifact # An escape peak
export SpectrumFeature # A CharXRay or EscapeArtifact (ComptonArtifact)
export EDSDetector # Abstract base for BasicEDS and other EDS detector-like things
export BasicEDS # A basic EDS detector with min visibility by line family
export channel # energy to channel
export isvisible # Is a SpectrumFeature visible in the spectrum??
export linewidth # energy to linewidth
export channelcount # Detector channel count
export apply # Create a copy of a spectrum with the specified detector
export scale # Detector EnergyScale
export resolution # Detector Resolution
export resolution_to_fwhm # Given resolution at E what is resolution at Mn Kα
export profile # Computes the resolution function
export simpleEDS # Create a basic EDS detector
export simpleEDSwICC # Create a basic EDS detector with a naive incomplete charge collection model
export extent # Determine the energy extent of an x-ray line on a detector ( Emin, Emax )
export extents # Determine the energy extents of many x-ray lines on a detector
export matching # Build a detector to match a spectrum
export matches # Do the spectrum and detector match
export lld # Low-level discriminator in eV
export detectorresponse # Build a matrix that describes the detectors response to X-rays
export gaussianwidth, fwhm

# X-ray window models
include("window.jl")
# primary function transmission(wnd, energy, angle)
export AbstractWindow, LayerWindow, TabulatedWindow
export AP33Model, AP5Model, AP33Tabulation, AP5Tabulation # Moxtek windows
export Beryllium # Classic windows
export AmptekC1, AmptekC2 # Amptek windows
export NoWindow # 100% transmission

include("detefficiency.jl")
export DetectorEfficiency
export efficiency
export SDDEfficiency, SiLiEfficiency # Helpers to build DetectorEfficiency
export buildresponse # Builds a detector response matrix

# Items defined in NeXL/spectrum.jl
include("spectrum.jl")
export Spectrum
#export NeXLCore.name # A human friendly name for the spectrum
export channel # channel for energy
export rangeofenergies # range of energies for channel `ch`
export channelwidth # width of channel ch
#export NeXLCore.energy # energy for channel
export dose # Spectrum probe dose
export property! # Set a property (easily broadcastable...)
export counts # Spectrum channel data
export integrate # Integrate range of channels
export kratio # Naive peak integration estimate of the k-ratio
export energyscale # Energy scale function
export subsample # Sub-sample a spectrum
export subdivide # Divide the counts among n spectra
export modelbackground # Model a background region
export modelBackground
export extractcharacteristic # Extract the characteristic intensity
export details # Outputs useful details about a spectrum
export peak # Estimates the peak intensity
export background # Estimates the background intensity
export estkratio # Estimate the k-ratio from two spectra for a ROI
export normalizedosewidth # Normalize intensity data to 1 nA⋅s⋅eV
export commonproperties # Properties that a collection of spectra share in common
export maxspectrum # Dave Bright's max spectra derived spectrum
export suitablefor # Which ROIs is a set of elements suitable for as reference for the specified element?
export missingReferences # A Vector with missing ROIs in a FilterFitPacket
export dosenormalize # Rescale the channel data to correspond to a different electron dose.

export maxproperty, minproperty # Min value of a property over a vector of spectra
export sameproperty # Returns the property value if all the spectra share the same value, errors otherwise
export textplot # A quick way to visualize a spectrum
export findsimilar # Find the spectra that are most similar to each other
export χ² # Compare spectra
export duane_hunt # Estimate the Duane-Hunt limit
export sigma # Computes the channel-by-channel dose corrected difference from the mean.

export recalibrate # Change the LinearEnergyScale against which a spectrum is calibrated
export shift # Shift the channels in a spectrum by a specified energy amount.
export offset # Offset the counts data by Δcounts

include("hyperspectrum.jl")
export HyperSpectrum # The wrapper that makes an Array{<:Real,N} look like an Array{Spectrum,N-1}
export plane # Sum planes in a HyperSpectrum
export roiimage # Convert a range of data channels into a Gray-scale image
export roiimages # Convert a vector of ranges-of-channels into Gray-scale images
export compress # Compresses Integer type data down to the smallest integer type that that will hold the max value.
export maxpixel, minpixel # Bright's max-pixel derived spectrum
export indexofmaxpixel # Index producing the max pixel
export avgpixel # Average intensity in a pixel
export sumcounts # Array containing the total number of counts in at each pixel
export depth # Number of spectral or result planes
export region # Extract a region from within the HyperSpectrum as a HyperSpectrum
export properties # The HyperSpectrum properties (mutable)
export axisname # The name of the i-th axis
export axisvalue # The calibrated coordinate value for the pixel coordinate
export axisrange # range of coordinate values for the specified axis
export livetime, livetime! # Get/Set livetime on a per-pixel basis
export block # Reduce the size of a HyperSpectrum by summing together blocks of adjacent pixels
export linescan # Extract the data in a 2D HyperSpectrum along a line (with width) as a 1D HyperSpectrum.

include("rplraw.jl")
export RPLHeader
export readrplraw # Read a RPL/RAW file into a Array
export readrpl # Read the RPL file into a RPLHeader
export writerplraw # Write a Array into a RPL/RAW

include("emsa.jl")
include("aspextiff.jl")
include("brukerpdz.jl")
include("brukerspx.jl")
include("fileiosupport.jl")
include("semanticsptx.jl")
include("hspy.jl")

export loadspectrum # Load a spectrum from IO or filename
export savespectrum # Save a spectrum to IO or filename to a format
export sniffspectrum # Determine spectrum file type
export isspectrumfile # Is a file a spectrum file?
export readptx # Read a SEMantics PTX file
export readhspy # Read a HyperSpy-style HDF5 file
export ishspy # Is the file a HyperSpy-style HDF5 file

# SpectrumFileType structs for `loadspectrum()` and `savespectrum()` support
export BrukerSPX # Read only, SEM/EDS & XRF format
export BrukerPDZ # Read only,  XRF format
export ISOEMSA   # Read / write, ISO/EMSA spectrum file format
export ASPEXTIFF # Read only, Vendor SEM/EDS format

include("fitlabels.jl")
export ReferenceLabel
export UnknownLabel#(spec) <: ReferenceLabel
export CharXRayLabel#(spec|props,roi,xrays) <: QuantifiableLabel
export EscapeLabel#(spec,roi,xrays) <: ReferenceLabel
export charXRayLabels # Constructs CharXRayLabel(s) <: ReferenceLabel

include("filter.jl")
export TopHatFilter # Struct representing a fitting filter
export VariableWidthFilter # The default filter definition that varies the filter width with x-ray energy
export ConstantWidthFilter # An alternative filter definition that holds the filter width constant
export GaussianFilter # An alternative filter definition based on an offset Gaussian
export tophatfilter # Apply a top-hat filter to produce a FilteredReference
export FilteredReference # A filtered datum representing a contiguous region of filtered reference data

export buildfilter
export estimatebackground
export ascontiguous
export filterfit
export filteredresidual
export filterreference
export filterreferences

include("fitresult.jl")
export FitResult
export BasicFitResult
export FilterFitResult
export findlabel
export fit_spectrum
export heterogeneity
export peaktobackground
export characteristiccounts
export residual
export spectrum
export unknown
export kratios
export kratio
export extractStandards

include("standardize.jl")
# export NeXLCore.standardize

include("llsq.jl")
# The implementation for weighted filter-fit.
include("filterfit_wls.jl")
export FilteredUnknownW # A filtered datum representing an unknown spectrum (for weighted least squares fitting)

include("continuum.jl")
export ContinuumModel # Build a model to compute the continuum intensity
export emitted # Compute the emitted continuum emission
export generated # Compute the generated continuum emission
export fitcontinuum # Fit a continuum model to a spectrum with :BeamEnergy, :TakeOffAngle, :Composition properties
export subtractcontinuum # Automatically remove the continuum contribution from a Spectrum
export fittedcontinuum
export continuumrois # The range of channels associated with the continuum

# These primarily exist to scale spectra for presentation (plotting)
include("specscaling.jl")
export SpectrumScaling # Abstract base
export NoScaling # <: SpectrumScaling
export ScaleDoseWidth # <: SpectrumScaling
export ScaleDose # <: SpectrumScaling
export ScaleSum # <: SpectrumScaling
export ScaleROISum # <: SpectrumScaling
export ScalePeak # <: SpectrumScaling
export ScaleWidth # <: SpectrumScaling
export scaledcounts # Return spectrum channel data scaled according the SpectrumScaling model

include("reference.jl")
export FilterFitPacket
export reference
export references
export spectra
export suitability # Tabulates material suitability for use as a fitting reference

include("qquant.jl")
export VectorQuant

include("labeled.jl")
export labeledimage # Displays an image and caption.
export labeledimages # Displays a grid of images and captions.
export plotandimage # Display an image the right of a Gadfly plot

include("line.jl")
export drawline
export drawray

include("quantify.jl")
export quantify

include("smoothing.jl")
export SavitzkyGolayFilter

include("multidet.jl") # Multi-detector support
export loadmultispec # Loads multiple related spectra.
export multiscore # Scores spectra relative to one another.
export multirank # A single number value that scores the spectra
export multisum, multimean
export multicompare

export plot_multicompare # Plots a channel-by-channel ratio comparison
export plot_compare # Plots a statistical channel-by-channel comparison of spectra

include("semantics_image.jl")
export readSEManticsImage
export annotate
export shannon_entropy

function __init__()
    @require Gadfly = "c91e804a-d5a3-530f-b6f0-dfbca275c004" include("gadflysupport.jl")
    @require Weave = "44d3d7a6-8a23-5bf8-98c5-b353f8df5ec9" include("weavesupport.jl")
end

end

ext: jl

max_stars_repo_name: NicholasWMRitchie/NeXLSpectrum

max_issues_repo_name: NicholasWMRitchie/NeXLSpectrum

max_forks_repo_name: NicholasWMRitchie/NeXLSpectrum


text:
import Pkg

const _create_dockerfile_kwargs_docstring = """
### Keyword Arguments (for all of the above methods)
- `additional_apt::Vector{String}`
- `exclude_packages_from_sysimage::Vector{String}`
- `julia_cpu_target::String`
- `julia_version::Union{String, VersionNumber}`
- `make_sysimage::Bool`
- `no_test::Vector{String}`
- `output_directory::String`. Default value: `pwd()`
- `parent_image::String`
- `tests_must_pass::Vector{String}`

### Advanced Keyword Arguments (for all of the above methods)
- `override_default_apt::Vector{String}`
- `packagecompiler_installation_command::String`
- `precompile_execution_env_vars::Dict{String, String}`
- `wrapper_script_env_vars::Dict{String, String}`
"""

"""
    create_dockerfile(config::SimpleContainerGenerator.Config;
                      output_directory = pwd())

### Arguments
- `config::SimpleContainerGenerator.Config`

### Keyword Arguments
- `output_directory::String`. Default value: `pwd()`
"""
function create_dockerfile(config::Config;
                           output_directory::AbstractString = pwd())
    template = Template(config)
    result = create_dockerfile(template;
                               output_directory = output_directory)
    return result
end

"""
    create_dockerfile(template::SimpleContainerGenerator.Template;
                      output_directory = pwd())

### Arguments
- `template::SimpleContainerGenerator.Template`

### Keyword Arguments
- `output_directory::String`. Default value: `pwd()`
"""
function create_dockerfile(template::Template;
                           output_directory::AbstractString = pwd())
    _write_all_docker_files(template;
                            output_directory = output_directory)
    @info("I have generated the Dockerfile and the other necessary files.")
    @info("Now `cd` to `$(output_directory)` and run:")
    @info("`docker build -t my_docker_username/my_image_name .`")
    return output_directory
end

"""
    create_dockerfile(pkg::Union{String, NamedTuple, Dict{Symbol, String}};
                      kwargs...)

    create_dockerfile(pkg_names::Vector{String};
                      kwargs...)

    create_dockerfile(pkg_tuples::Vector{<:NamedTuple};
                      kwargs...)

    create_dockerfile(pkgs::Vector{Dict};
                      kwargs...)

$(_create_dockerfile_kwargs_docstring)
"""
function create_dockerfile(varargs...;
                           output_directory::AbstractString = pwd(),
                           kwargs...)
    config = Config(varargs...; kwargs...)
    result = create_dockerfile(config;
                               output_directory = output_directory)
    return result
end


text:
__precompile__(true)
module NOAAData

using HTTP
using DataFrames
using IndexedTables
using Dates
using JSON

import Base.string, Base.get
import DataFrames.DataFrame
import IndexedTables.NDSparse

export NOAA, GHCND, GSOM, get, DataFrame, NDSparse, getstations, getlocations

struct NOAA
  token::String
end

abstract type NOAADataSet end

struct GHCND <: NOAADataSet end
struct GSOM <: NOAADataSet end

_divide_10(x::Float64) = x / 10.0
_divide_100(x::Float64) = x / 100.0
const SCHEMAS = Dict{String, Tuple{Vector{Union{DataType, Union}}, Vector{Symbol}}}(
  "GHCND" => ([Date, Union{Float64, Nothing}, DateTime, DateTime, Union{Float64, Nothing}, Union{Float64, Nothing},
              Union{Float64, Nothing}, Union{Float64, Nothing}, Union{Float64, Nothing}, Union{Float64, Nothing},
              Union{Float64, Nothing}, Union{Float64, Nothing}, Union{Float64, Nothing}, Union{Float64, Nothing}, Union{String, Nothing}],
              [:DATE, :AWND, :FMTM, :PGTM, :PRCP, :SNOW, :SNWD, :TMAX, :TMIN, :TAVG, :WDF2, :WDF5, :WSF2, :WSF5, :WT]),
  "GSOM" => ([Date, Union{Float64, Nothing}, Union{Int, Nothing}, Union{Int, Nothing}, Union{Int, Nothing}, Union{Int, Nothing},
              Union{Int, Nothing}, Union{Int, Nothing}, Union{Int, Nothing}, Union{Int, Nothing}, Union{Int, Nothing},
              Union{Int, Nothing}, Union{Int, Nothing}, Union{Int, Nothing}, Union{Int, Nothing}, Union{Float64, Nothing},
              Union{Int, Nothing}, Union{Int, Nothing}, Union{Float64, Nothing}, Union{Float64, Nothing}, Union{Float64, Nothing},
              Union{Float64, Nothing}, Union{Float64, Nothing}, Union{Int, Nothing}, Union{Float64, Nothing}, Union{Float64, Nothing},
              Union{Float64, Nothing}, Union{Float64, Nothing}, Union{Int, Nothing}, Union{Int, Nothing}, Union{Float64, Nothing},
              Union{Int, Nothing}, Union{Float64, Nothing}, Union{Int, Nothing}, Union{Int, Nothing}, Union{Int, Nothing},
              Union{Float64, Nothing}, Union{Float64, Nothing}, Union{Float64, Nothing}],
              [:DATE, :AWND, :CDSD, :CLDD, :DP01, :DP05, :DP10, :DP1X, :DSND, :DSNW, :DT00, :DT32, :DX32, :DX70, :DX90, :EMNT, :EMSD, :EMSN, :EMXP,
              :EMXT, :HDSD, :HTDD, :PRCP, :SNOW, :TAVG, :TMAX, :TMIN, :PSUN, :TSUN, :WDFM, :WSFM, :WDFG, :WSFG, :WDF1, :WDF2, :WDF5, :WSF1, :WSF2, :WSF5])
)

const CONVERTER_GHCND = Dict{Symbol, Function}(
  :TMAX => _divide_10,
  :TMIN => _divide_10,
  :TAVG => _divide_10,
  # :SNOW => _divide_10,
  :PRCP => _divide_10
)

const CONVERTER_GSOM = Dict{Symbol, Function}()

_get_converter(::GHCND) = CONVERTER_GHCND
_get_converter(::GSOM) = CONVERTER_GSOM

const DATETIMEFORMAT = Dates.DateFormat("y-m-d HHMM")
const DATEFORMAT = Dates.DateFormat("y-m-d")

_flt_identity(v::Float64) = v
_flt_identity(::Nothing) = nothing
# _defaultval(::Type{Float64}, ::Date) = 0.0
# _defaultval(::Type{Int}, ::Date) = 0
# _defaultval(::Type{String}, ::Date) = ""
_defaultval(::Type, ::Date) = nothing
_defaultval(::Type{Date}, d::Date) = d
_defaultval(::Type{DateTime}, d::Date) = DateTime(d)

_conversion(::Type{Union{Nothing, String}}, ::NOAADataSet, v::Dict, ::Symbol) = v["datatype"]

_flt_conversion(v) = float(v)
_flt_conversion(v::Nothing) = nothing
function _conversion(::Type{Union{Nothing, Float64}}, ds::NOAADataSet, v::Dict, symb::Symbol)
  return get(_get_converter(ds), symb, _flt_identity)(_flt_conversion(v["value"]))
end

_int_conversion(v) = parse(Int, v)
_int_conversion(v::Int) = v
_int_conversion(v::Float64) = round(Int, v)
_int_conversion(v::Nothing) = nothing
_conversion(::Type{Union{Nothing, Int}}, ::NOAADataSet, v::Dict, ::Symbol) = _int_conversion(v["value"])

function _conversion(::Type{DateTime}, ::NOAADataSet, v::Dict, ::Symbol)
  tm = string(v["value"])
  if length(tm) == 3
    tm = "0" * tm
  end
  currdate = split(v["date"], "T")[1]
  return DateTime(currdate * " " * tm, DATETIMEFORMAT)
end

extendval(existingval::String, val::String) = existingval * " " * val

_get_schema(::GHCND) = SCHEMAS["GHCND"]
_get_schema(::GSOM) = SCHEMAS["GSOM"]

_check_date_range(::GHCND, d1::Date, d2::Date) = (d2 - d1).value < 366
_check_date_range(::GSOM, d1::Date, d2::Date) = (d2 - d1).value < 1830

function check_date_range(ds::NOAADataSet, d1::Date, d2::Date)
  if d1 > d2
    return false
  end
  return _check_date_range(ds, d1, d2)
end

string(::GHCND) = "GHCND"
string(::GSOM) = "GSOM"

struct NOAADataResult{D <: NOAADataSet}
  data::Vector
  count::Int
  dataset::D
  stationid::String
end

function getresults(baseurl::String, headers::Dict{String, String}, query::Dict{String, String})
  result = []
  offset = 1
  while true
    resp = HTTP.get(baseurl, headers; query=query)
    js = JSON.parse(String(resp.body))
    if length(js["results"]) == 1000
      # we probably maxed out, need to get a new query
      append!(result, js["results"])
      offset += 1000
      query["offset"] = string(offset)
    else
      append!(result, js["results"])
      break
    end
  end

  return result
end

function get(ds::NOAADataSet, noaa::NOAA, startdate::Date, enddate::Date, stationid::String)
  check_date_range(ds, startdate, enddate) || error("Invalid date range")
  baseurl = "https://www.ncdc.noaa.gov/cdo-web/api/v2/data"
  query =  Dict{String, String}()
  query["datasetid"] = string(ds)
  query["stationid"] = stationid
  query["startdate"] = string(startdate)
  query["enddate"] = string(enddate)
  query["limit"] = "1000"
  headers = Dict{String, String}()
  headers["token"] = noaa.token

  result = getresults(baseurl, headers, query)

  return NOAADataResult(result, length(result), ds, stationid)
end

function _process_data(result::NOAADataResult)
  schema = _get_schema(result.dataset)
  data = result.data
  cols = NamedTuple{Tuple(schema[2])}(dt[] for dt =schema[1])
  i = 1
  rowiter = 1
  startingdate = Date(split(data[i]["date"], "T")[1], DATEFORMAT)
  while i <= result.count
    currdate = Date(split(data[i]["date"], "T")[1], DATEFORMAT)
    if currdate != startingdate
      # check for blanks
      for col = cols
        if length(col) != rowiter
          push!(col, _defaultval(eltype(col), startingdate))
        end
      end
      startingdate = currdate
      rowiter += 1
    end
    coliter = Symbol(data[i]["datatype"])
    coltoadd = get(cols, coliter, cols[end])
    val = _conversion(eltype(coltoadd), result.dataset, data[i], coliter)
    if length(coltoadd) == rowiter
      # value already exists
      coltoadd[rowiter] = extendval(coltoadd[rowiter], val)
    else
      push!(coltoadd, val)
    end
    i += 1
  end
  # final cleanup
  for col = cols
    if length(col) != rowiter
      push!(col, _defaultval(eltype(col), startingdate))
    end
  end
  return cols, schema
end

function NDSparse(result::NOAADataResult)
  cols, schema = _process_data(result)
  return NDSparse(Columns(cols[1]; names=schema[2][1:1]), Columns(cols[2:end]...; names=schema[2][2:end]))
end

function DataFrame(result::NOAADataResult)
  cols, schema = _process_data(result)
  return DataFrame(cols, schema[2])
end

## station queries

function getstations(
    noaa::NOAA;
    dataset::Union{NOAADataSet, Nothing} = nothing,
    locationid::Union{String, Nothing} = nothing,
    extent::Union{Vector{Float64}, Nothing} = nothing,
    startdate::Union{Date, Nothing} = nothing,
    enddate::Union{Date, Nothing} = nothing,
    datacategoryid::Union{String, Nothing} = nothing
  )
  baseurl = "https://www.ncdc.noaa.gov/cdo-web/api/v2/stations"
  query = Dict{String, String}()
  if dataset != nothing
    query["datasetid"] = string(dataset)
  end

  if locationid != nothing
    query["locationid"] = locationid
  end

  if extent != nothing
    query["extent"] = join(extent, ",")
  end

  if startdate != nothing
    query["startdate"] = string(startdate)
  end

  if enddate != nothing
    query["enddate"] = string(enddate)
  end

  if datacategoryid != nothing
    query["datacategoryid"] = datacategoryid
  end

  query["limit"] = "1000"

  headers = Dict{String, String}()
  headers["token"] = noaa.token

  result = getresults(baseurl, headers, query)

  # build dataframe from list of dicts
  DataFrame(
    id = getindex.(result, "id"),
    name = getindex.(result, "name"),
    latitude = getindex.(result, "latitude"),
    longitude = getindex.(result, "longitude"),
    mindate = Date.(getindex.(result, "mindate")),
    maxdate = Date.(getindex.(result, "maxdate")),
    elevation = getindex.(result, "elevation"),
    elevation_unit = getindex.(result, "elevationUnit"),
    datacoverage = getindex.(result, "datacoverage")
  )
end

## location queries

function getlocations(
    noaa::NOAA;
    dataset::Union{NOAADataSet, Nothing} = nothing,
    locationcategoryid::Union{String, Nothing} = nothing,
    extent::Union{Vector{Float64}, Nothing} = nothing,
    startdate::Union{Date, Nothing} = nothing,
    enddate::Union{Date, Nothing} = nothing,
    datacategoryid::Union{String, Nothing} = nothing
  )
  baseurl = "https://www.ncdc.noaa.gov/cdo-web/api/v2/locations"
  query = Dict{String, String}()
  if dataset != nothing
    query["datasetid"] = string(dataset)
  end

  if locationcategoryid != nothing
    query["locationcategoryid"] = locationcategoryid
  end

  if extent != nothing
    query["extent"] = join(extent, ",")
  end

  if startdate != nothing
    query["startdate"] = string(startdate)
  end

  if enddate != nothing
    query["enddate"] = string(enddate)
  end

  if datacategoryid != nothing
    query["datacategoryid"] = datacategoryid
  end

  query["limit"] = "1000"

  headers = Dict{String, String}()
  headers["token"] = noaa.token

  result = getresults(baseurl, headers, query)

  # build dataframe from list of dicts
  DataFrame(
    id = getindex.(result, "id"),
    name = getindex.(result, "name"),
    mindate = Date.(getindex.(result, "mindate")),
    maxdate = Date.(getindex.(result, "maxdate")),
    datacoverage = getindex.(result, "datacoverage")
  )
end

end # module


text:
module Xu

@doc raw"""
    f(n,ℓ)

```math
f(n,\ell)=\begin{cases}
\displaystyle\sum_{m=0}^n \ell-m, & n\geq0\\
0, & n<0
\end{cases}
```
"""
f(n::I,ℓ::I) where {I<:Integer} = n >= 0 ? sum(ℓ-m for m in 0:n) : 0

@doc raw"""
``A(N,\ell,\ell_b,M_S',M_L)`` obeys four different cases:

## Case 1

``M_S'=1``, ``|M_L|\leq\ell``, and ``N=1``:

```math
A(1,\ell,\ell_b,1,M_L) = 1
```

## Case 2

``\{M_S'\}={2-N,4-N,...,N-2}``, ``|M_L| \leq f\left(\frac{N-M_S'}{2}-1\right)+f\left(\frac{N+M_S'}{2}-1\right)``, and ``1<N\leq 2\ell+1``:

```math
\begin{aligned}
A(N,\ell,\ell,M_S',M_L) =
\sum_{M_{L-}\max\left\{-f\left(\frac{N-M_S'}{2}-1\right),M_L-f\left(\frac{N+M_S'}{2}-1\right)\right\}}
^{\min\left\{f\left(\frac{N-M_S'}{2}-1\right),M_L+f\left(\frac{N+M_S'}{2}-1\right)\right\}}
\Bigg\{A\left(\frac{N-M_S'}{2},\ell,\ell,\frac{N-M_S'}{2},M_{L-}\right)\\
\times
A\left(\frac{N+M_S'}{2},\ell,\ell,\frac{N+M_S'}{2},M_L-M_{L-}\right)\Bigg\}
\end{aligned}
```

## Case 3

``M_S'=N``, ``|M_L|\leq f(N-1)``, and ``1<N\leq 2\ell+1``:

```math
A(N,\ell,\ell_b,N,M_L) =
\sum_{M_{L_I} = \left\lfloor{\frac{M_L-1}{N}+\frac{N+1}{2}}\right\rfloor}
^{\min\{\ell_b,M_L+f(N-2)\}}
A(N-1,\ell,M_{L_I}-1,N-1,M_L-M_{L_I})
```

## Case 4

else:

```math
A(N,\ell,\ell_b,M_S',M_L) = 0
```
"""
function A(N::I, ℓ::I, ℓ_b::I, M_S′::I, M_L::I) where {I<:Integer}
    if M_S′ == 1 && abs(M_L) <= ℓ && N == 1
        1 # Case 1
    elseif 1 < N && N <= 2ℓ + 1
        # Cases 2 and 3
        # N ± M_S' is always an even number
        a = (N-M_S′) >> 1
        b = (N+M_S′) >> 1
        fa = f(a-1,ℓ)
        fb = f(b-1,ℓ)

        if M_S′ in 2-N:2:N-2 && abs(M_L) <= fa + fb
            mapreduce(+, max(-fa, M_L - fb):min(fa, M_L + fb)) do M_Lm
                A(a,ℓ,ℓ,a,M_Lm)*A(b,ℓ,ℓ,b,M_L-M_Lm)
            end
        elseif M_S′ == N && abs(M_L) <= f(N-1,ℓ)
            mapreduce(+, floor(Int, (M_L-1)//N + (N+1)//2):min(ℓ_b,M_L + f(N-2,ℓ))) do M_LI
                A(N - 1, ℓ, M_LI - 1, N - 1, M_L-M_LI)
            end
        else
            0 # Case 4
        end
    else
        0 # Case 4
    end
end

@doc raw"""
    X(N, ℓ, S′, L)

Calculate the multiplicity of the term ``^{2S+1}L`` (``S'=2S``) for
the orbital `ℓ` with occupancy `N`, according to the formula:

```math
\begin{aligned}
X(N, \ell, S', L) =&+ A(N, \ell,\ell,S',L)\\
&- A(N, \ell,\ell,S',L+1)\\
&+A(N, \ell,\ell,S'+2,L+1)\\
&- A(N, \ell,\ell,S'+2,L)
\end{aligned}
```

Note that this is not correct for filled (empty) shells, for which the
only possible term trivially is `¹S`.

# Examples

```jldoctest
julia> AtomicLevels.Xu.X(1, 0, 1, 0) # Multiplicity of ²S term for s¹
1

julia> AtomicLevels.Xu.X(3, 3, 1, 3) # Multiplicity of ²D term for d³
2
```

"""
X(N::I, ℓ::I, S′::I, L::I) where {I<:Integer} = A(N,ℓ,ℓ,S′,L) - A(N,ℓ,ℓ,S′,L+1) + A(N,ℓ,ℓ,S′+2,L+1) - A(N,ℓ,ℓ,S′+2,L)

end


text:
module DDM

end

max_stars_repo_name: WaveProp/WaveProp

max_issues_repo_name: WaveProp/WaveProp

max_forks_repo_name: WaveProp/WaveProp


text:
import Oceananigans: summary
using Oceananigans.Fields: show_location

for op_string in ("UnaryOperation", "BinaryOperation", "MultiaryOperation", "Derivative", "KernelFunctionOperation")
    op = eval(Symbol(op_string))
    @eval begin
        operation_name(::$op) = $op_string
    end
end

operation_name(op::GridMetricOperation)  = string(op.metric)

function show_interp(op)
    op_str = string(op)
    if length(op_str) >= 8 && op_str[1:8] == "identity"
        return "identity"
    else
        return op_str
    end
end

Base.summary(operation::AbstractOperation) = string(operation_name(operation), " at ", show_location(operation))

Base.show(io::IO, operation::AbstractOperation) =
    print(io,
          summary(operation), '\n',
          "├── grid: ", summary(operation.grid), '\n',
          "└── tree: ", "\n", "    ", tree_show(operation, 1, 0))

"Return a representation of number or function leaf within a tree visualization of an `AbstractOperation`."
tree_show(a::Union{Number, Function}, depth, nesting) = string(a)

"Fallback for displaying a leaf within a tree visualization of an `AbstractOperation`."
tree_show(a, depth, nesting) = summary(a) # fallback

"Returns a string corresponding to padding characters for a tree visualization of an `AbstractOperation`."
get_tree_padding(depth, nesting) = "    "^(depth-nesting) * "│   "^nesting

"Return a string representaion of a `UnaryOperation` leaf within a tree visualization of an `AbstractOperation`."
function tree_show(unary::UnaryOperation, depth, nesting)
    padding = get_tree_padding(depth, nesting)
    LX, LY, LZ = location(unary)

    return string(unary.op, " at ", show_location(LX, LY, LZ), " via ", show_interp(unary.▶), '\n',
                  padding, "└── ", tree_show(unary.arg, depth+1, nesting))
end

"Return a string representaion of a `BinaryOperation` leaf within a tree visualization of an `AbstractOperation`."
function tree_show(binary::BinaryOperation, depth, nesting)
    padding = get_tree_padding(depth, nesting)
    LX, LY, LZ = location(binary)

    return string(binary.op, " at ", show_location(LX, LY, LZ), '\n',
                  padding, "├── ", tree_show(binary.a, depth+1, nesting+1), '\n',
                  padding, "└── ", tree_show(binary.b, depth+1, nesting))
end

"Return a string representaion of a `MultiaryOperation` leaf within a tree visualization of an `AbstractOperation`."
function tree_show(multiary::MultiaryOperation, depth, nesting)
    padding = get_tree_padding(depth, nesting)
    LX, LY, LZ = location(multiary)
    N = length(multiary.args)

    out = string(multiary.op, " at ", show_location(LX, LY, LZ), '\n',
        ntuple(i -> padding * "├── " * tree_show(multiary.args[i], depth+1, nesting+1) * '\n', Val(N-1))...,
                    padding * "└── " * tree_show(multiary.args[N], depth+1, nesting))
    return out
end

"Return a string representaion of a `Derivative` leaf within a tree visualization of an `AbstractOperation`."
function tree_show(deriv::Derivative, depth, nesting)
    padding = get_tree_padding(depth, nesting)
    LX, LY, LZ = location(deriv)

    return string(deriv.∂, " at ", show_location(LX, LY, LZ), " via ", show_interp(deriv.▶), '\n',
                  padding, "└── ", tree_show(deriv.arg, depth+1, nesting))
end


text:
using Base: Symbol
"""

    plot_posterior_intervals(model, point_est, prob, args...; kwargs...)

Draw a plot with a point estimate measure of centrality and quantiled credible intervals for easy interpretation of models fit in PPLs such as `Turing.jl` or `Soss.jl`.

Usage:
```julia-repl
plot_posterior_intervals(model, point_est, prob)
```

Arguments:

- `model` : The model to draw inferences from.
- `point_est` : The type of point estimate to use.
- `prob` : The probability of the credible interval to calculate.
"""
function plot_posterior_intervals(model, point_est::String = "median", prob::Float64 = 0.95, args...; kwargs...)

    # Fix seed for reproducibility

    Random.seed!(123)

    # Check point estimate argument

    (point_est == "mean" || point_est == "median") || error("`point_est` should be a String of either 'mean' or 'median'.")

    # Check prob argument

    prob > 0 || error("`prob` should be a single Float64 value of 0 < prob < 1.")
    prob < 1 || error("`prob` should be a single Float64 value of 0 < prob < 1.")
    quantileRange = generatequantile(prob)

    # Turn model objects into DataFrames

    if isa(model, Array)

        myarray = DataFrame(model)

        #-------- Wrangle floats --------

        # Reshape

        thefloats = select(myarray, findall(col -> eltype(col) <: Float64, eachcol(myarray)))
        ncols = size(thefloats, 2)
        stackedfloats = stack(thefloats, 1:ncols)

        # Point estimate and credible intervals

        if point_est == median
            finalfloats = combine(groupby(stackedfloats, :variable), :value => median => :centre, :value => (t -> quantile(t, quantileRange[1])) => :lower, :value => (t -> quantile(t, quantileRange[2])) => :upper)
        else
            finalfloats = combine(groupby(stackedfloats, :variable), :value => mean => :centre, :value => (t -> quantile(t, quantileRange[1])) => :lower, :value => (t -> quantile(t, quantileRange[2])) => :upper)
        end

        #-------- Wrangle arrays --------

        # Reshape

        thearrays = select(myarray, findall(col -> eltype(col) <: Array, eachcol(myarray)))
        ncols_arrays = size(thearrays, 2)

        if ncols_arrays > 1
            error("`plot_posterior_intervals` currently only works for models where an array of parameters exists for one type of parameter (e.g. β coefficients stored in the array). Other parameters can be stored as regular Float64 columns.")
        else

            # Parse columns and retain only the separated ones

            parsedarrays = expandcol(thearrays, Symbol(names(thearrays)[1]))

            parsedarraysCols = select(parsedarrays, findall(col -> eltype(col) <: Float64, eachcol(parsedarrays)))

            # Reshape

            ncols_res_arrays = size(parsedarraysCols, 2)
            stackedarraysfloats = stack(parsedarraysCols, 1:ncols_res_arrays)

            # Point estimate and credible intervals

            if point_est == median
                finalfloats = combine(groupby(stackedarraysfloats, :variable), :value => median => :centre, :value => (t -> quantile(t, quantileRange[1])) => :lower, :value => (t -> quantile(t, quantileRange[2])) => :upper)
            else
                finalfloats = combine(groupby(stackedarraysfloats, :variable), :value => mean => :centre, :value => (t -> quantile(t, quantileRange[1])) => :lower, :value => (t -> quantile(t, quantileRange[2])) => :upper)
            end
        end

        #-------- Final outputs ---------

        # Merge together

        finalPost = [finalfloats; finalfloatsarrays]
        finalPost = rename(finalPost, :variable => :parameters)

        # Standardise outputs

        variable = finalPost[!, :parameters]
        variable = string.(variable)
        lower = finalPost[!, :lower]
        centre = finalPost[!, :centre]
        upper = finalPost[!, :upper]

    elseif isa(model, Chains)

        # Extract values

        finalPost = DataFrame(MCMCChains.quantile(model; q = [quantileRange[1], 0.50, quantileRange[2]]))
        my_names = ["parameters", "lower", "centre", "upper"]
        finalPost = rename!(finalPost, my_names)

        # Standardise outputs

        variable = finalPost[!, :parameters]
        variable = string.(variable)
        lower = finalPost[!, :lower]
        centre = finalPost[!, :centre]
        upper = finalPost[!, :upper]

    elseif isa(model, MultiChain)

        # Wrangle into tidy format

        finalPost = DataFrame(model)
        ncols = size(finalPost, 2)
        finalPost = stack(finalPost, 1:ncols)

        if point_est == median
            finalfloats = combine(groupby(finalPost, :variable), :value => median => :centre, :value => (t -> quantile(t, quantileRange[1])) => :lower, :value => (t -> quantile(t, quantileRange[2])) => :upper)
        else
            finalfloats = combine(groupby(finalPost, :variable), :value => mean => :centre, :value => (t -> quantile(t, quantileRange[1])) => :lower, :value => (t -> quantile(t, quantileRange[2])) => :upper)
        end

        # Standardise outputs

        finalPost = rename(finalPost, :variable => :parameters)
        variable = finalPost[!, :parameters]
        variable = string.(variable)
        lower = finalPost[!, :lower]
        centre = finalPost[!, :centre]
        upper = finalPost[!, :upper]
    else
        error("`model` must be an object of type `Chains`, `Array`, or `MultiChain`.")
    end

    #------------ Draw the plot -----------------

    gr() # gr backend for graphics
    centrecolour = cgrad(:blues)[2]

    myPlot = plot(centre, variable, xerror = (centre .- lower, upper .- centre), st = :scatter,
                title = string("Posterior ", point_est, "s with credible intervals"),
                xlabel = "Value",
                ylabel = "Parameter",
                legend = false,
                markersize = 5,
                color = centrecolour, markerstrokecolor = centrecolour,
                size = (600, 600))

    return myPlot
end

"""

    plot_posterior_hist(model, point_est, plot_legend, args...; kwargs...)

Draw a plot with a binned histogram of sampled parameters for easy interpretation of models fit in PPLs such as `Turing.jl` or `Soss.jl`.

Usage:
```julia-repl
plot_posterior_hist(model, point_est, plot_legend)
```

Details:

Note that to get the function to work, you may need to call it using a splat, such as `plot(plot_posterior_hist(model, plot_legend)...)`.

Arguments:

- `model` : The model to draw inferences from.
- `point_est` : The type of point estimate to use.
- `plot_legend` : Boolean of whether to add a legend to the plot or not.
"""
function plot_posterior_hist(model, point_est::String = "median", plot_legend::Bool = true, args...; kwargs...)

    # Check point estimate argument

    (point_est == "mean" || point_est == "median") || error("`point_est` should be a String of either 'mean' or 'median'.")

    if isa(model, Array)

        myarray = DataFrame(model)

        #-------- Wrangle floats --------

        thefloats = select(myarray, findall(col -> eltype(col) <: Float64, eachcol(myarray)))

        #-------- Wrangle arrays --------

        thearrays = select(myarray, findall(col -> eltype(col) <: Array, eachcol(myarray)))
        ncols_arrays = size(thearrays, 2)

        if ncols_arrays > 1
            error("`plot_posterior_intervals` currently only works for models where an array of parameters exists for one type of parameter (e.g. β coefficients stored in the array). Other parameters can be stored as regular Float64 columns.")
        else

            # Parse columns and retain only the separated ones

            parsedarrays = expandcol(thearrays, Symbol(names(thearrays)[1]))

            parsedarraysCols = select(parsedarrays, findall(col -> eltype(col) <: Float64, eachcol(parsedarrays)))
        end

        #-------- Merge together --------

        posteriorDF = [thefloats parsedarraysCols]
        params = propertynames(posteriorDF)

    elseif isa(model, Chains)

        posteriorDF = DataFrame(model)

        # Retrieve just the model parameter names to avoid the sampler specific columns

        paramsData = DataFrame(MCMCChains.summarize(model))
        params = paramsData[!, :parameters]

    elseif isa(model, MultiChain)

        posteriorDF = DataFrame(model)

    else
        error("`model` must be an object of type `Chains`, `Array`, or `MultiChain`.")
    end

    #------------ Draw the plots ----------------

    gr() # gr backend for graphics

    # Draw plot for each parameter

    myPlotArray = [histhelper(posteriorDF, point_est, p, plot_legend) for p in params]

    return myPlotArray
end

"""

    plot_posterior_density(model, point_est, prob, plot_legend, args...; kwargs...)

Draw a density plot of sampled parameters for easy interpretation of models fit in PPLs such as `Turing.jl` or `Soss.jl`.

Usage:
```julia-repl
plot_posterior_density(model, point_est, prob, plot_legend)
```

Details:

Note that to get the function to work, you may need to call it using a splat, such as `plot(plot_posterior_density(model, point_est, plot_legend)...)`.

Arguments:

- `model` : The model to draw inferences from.
- `point_est` : The type of point estimate to use.
- `prob` : The probability of the credible interval to calculate.
- `plot_legend` : Boolean of whether to add a legend to the plot or not.
"""
function plot_posterior_density(model, point_est::String = "median", prob::Float64 = 0.95, plot_legend::Bool = true, args...; kwargs...)

    # Check point estimate argument

    (point_est == "mean" || point_est == "median") || error("`point_est` should be a String of either 'mean' or 'median'.")

    # Check prob argument

    prob > 0 || error("`prob` should be a single Float64 value of 0 < prob < 1.")
    prob < 1 || error("`prob` should be a single Float64 value of 0 < prob < 1.")
    quantileRange = generatequantile(prob)

    if isa(model, Array)

        myarray = DataFrame(model)

        #-------- Wrangle floats --------

        thefloats = select(myarray, findall(col -> eltype(col) <: Float64, eachcol(myarray)))

        #-------- Wrangle arrays --------

        thearrays = select(myarray, findall(col -> eltype(col) <: Array, eachcol(myarray)))
        ncols_arrays = size(thearrays, 2)

        if ncols_arrays > 1
            error("`plot_posterior_intervals` currently only works for models where an array of parameters exists for one type of parameter (e.g. β coefficients stored in the array). Other parameters can be stored as regular Float64 columns.")
        else

            # Parse columns and retain only the separated ones

            parsedarrays = expandcol(thearrays, Symbol(names(thearrays)[1]))

            parsedarraysCols = select(parsedarrays, findall(col -> eltype(col) <: Float64, eachcol(parsedarrays)))
        end

        #-------- Merge together --------

        posteriorDF = [thefloats parsedarraysCols]
        params = propertynames(posteriorDF)

    elseif isa(model, Chains)

        posteriorDF = DataFrame(model)

        # Retrieve just the model parameter names to avoid the sampler specific columns

        paramsData = DataFrame(MCMCChains.summarize(model))
        params = paramsData[!, :parameters]

    elseif isa(model, MultiChain)

        posteriorDF = DataFrame(model)

    else
        error("`model` must be an object of type `Chains`, `Array`, or `MultiChain`.")
    end

    #------------ Draw the plots ----------------

    gr() # gr backend for graphics

    # Draw plot for each parameter

    myPlotArray = [denshelper(posteriorDF, point_est, p, quantileRange[1], quantileRange[2], plot_legend) for p in params]

    return myPlotArray
end


text:
using Test
using MeshArrays

@testset "GCMFaces tests:" begin
for nTopo=1:3
    if nTopo==1; grTopo="cs"; nFaces=6; N=200;
    elseif nTopo==2; grTopo="llc"; nFaces=5; N=200;
    elseif nTopo==3; grTopo="ll"; nFaces=1; N=500;
    end;
    Npt=nFaces*N*N
    @test GCMGridOnes(grTopo,nFaces,N) == "GCMGridOnes: passed"
    Rini= 0.; Rend= 0.;
    (Rini,Rend,DXCsm,DYCsm)=demo2();
    @test isa(Rend,gcmfaces)
    @test sum(isfinite(Rend)) == Npt
    Sini=sqrt(sum(Rini*Rini)/(Npt-1.0))
    Send=sqrt(sum(Rend*Rend)/(Npt-1.0))
    #println([Sini Send])
    @test isapprox(Sini,1.000; atol=1e-2)
    @test isapprox(Send,0.093; atol=1e-2)
end;
end;


text:
# Note that this script can accept some limited command-line arguments, run
# `julia build_tarballs.jl --help` to see a usage message.
using BinaryBuilder, Pkg

name = "Xyce"
version = v"7.4.99"

# Collection of sources required to complete build
sources = [
            GitSource("https://github.com/Xyce/Xyce.git", "b7bb12d81f11d8b50141262537299b09d64b5565"),
            DirectorySource("./bundled")
          ]

# Bash recipe for building across all platforms
script = raw"""
export TMPDIR=${WORKSPACE}/tmpdir
mkdir ${TMPDIR}
cd $WORKSPACE/srcdir
apk add flex-dev
update_configure_scripts --reconf
install_license ${WORKSPACE}/srcdir/Xyce/COPYING
cd Xyce
for f in ${WORKSPACE}/srcdir/patches/*.patch; do
    atomic_patch -p1 ${f}
done
./bootstrap
cd ..
mkdir buildx
cd buildx
/workspace/srcdir/Xyce/./configure --prefix=${prefix} --build=${MACHTYPE} --host=${target} \
    --enable-shared --disable-mpi \
    LDFLAGS="-L${libdir} -lopenblas" \
    CPPFLAGS="-I/${includedir} -I/usr/include"
make -j${nprocs}
make install
"""

# These are the platforms we will build for by default, unless further
# platforms are passed in on the command line

platforms = supported_platforms()

platforms = expand_cxxstring_abis(platforms)

# The products that we will ensure are always built
products = [
    LibraryProduct("libxyce", :libxyce),
    ExecutableProduct("Xyce", :Xyce)
]

# Dependencies that must be installed before this package can be built
dependencies = [
                    Dependency(PackageSpec(name="Trilinos_jll", uuid="b6fd3212-6f87-5999-b9ea-021e9cd21b17"))
                    Dependency(PackageSpec(name="SuiteSparse_jll", uuid="bea87d4a-7f5b-5778-9afe-8cc45184846c"))
                    Dependency(PackageSpec(name="OpenBLAS32_jll", uuid="656ef2d0-ae68-5445-9ca0-591084a874a2"))
                    Dependency(PackageSpec(name="FFTW_jll", uuid="f5851436-0d7a-5f13-b9de-f02708fd171a"))
                    Dependency(PackageSpec(name="CompilerSupportLibraries_jll", uuid="e66e0078-7015-5450-92f7-15fbd957f2ae"))
                ]

# Build the tarballs, and possibly a `build.jl` as well.
build_tarballs(ARGS, name, version, sources, script, platforms, products, dependencies; preferred_gcc_version = v"7.1.0")

max_stars_repo_name: sharanry/Yggdrasil

max_issues_repo_name: sharanry/Yggdrasil

max_forks_repo_name: sharanry/Yggdrasil


text:
# Set up environment for Julia Windows binary distribution
ENV["PATH"] = JULIA_HOME*";"*ENV["PATH"]


text:
if typeof(Pkg.installed("CUDArt")) != VersionNumber
   Pkg.add("CUDArt")
end

if !contains(ENV["PATH"],"cuda")
  ENV["PATH"] = string(ENV["PATH"],":/usr/local/cuda/bin")
end

if !( mtime("normal_pdf_kernel.ptx") >= mtime("normal_pdf_kernel.cu") )
  run(`nvcc -ptx normal_pdf_kernel.cu`)
end

using CUDArt

function init_cuda(gpuid::Integer = first(devices(dev->capability(dev)[1]>=2)) )
  dev = device(gpuid) # specify a CUDA device, defaulting to the first GPU with CUDA compute capability >=2.0

  # Set parameters based on the attributes of this specific GPU
  const global max_grid_dimx_size = attribute(dev,CUDArt.CUDArt_gen.cudaDevAttrMaxGridDimX )
  const global max_block_dimx_size = attribute(dev,CUDArt.CUDArt_gen.cudaDevAttrMaxBlockDimX )
  const global max_threads_per_block = attribute(dev,CUDArt.CUDArt_gen.cudaDevAttrMaxThreadsPerBlock)

  return dev
end

function load_functions(T::Type=Float64)
  # load the PTX module containing compiled GPU kernels (each module can contain multiple kernel functions)
  md = CuModule("normal_pdf_kernel.ptx", false)  # false specifies not to finalize module

  # Retrieve a few kernel functions from the module
  if T==Float64
    global std_normal_pdf_kernel = CuFunction(md, "std_normal_pdf_double")
    global normal_pdf_kernel = CuFunction(md, "normal_pdf_double")
    global block_sum_kernel = CuFunction(md, "block_sum_double")
    global sum_kernel = CuFunction(md, "sum_simplistic_double")
#=
  elseif T==Float32
    global std_normal_pdf_kernel = CuFunction(md, "std_normal_pdf_float")
    global normal_pdf_kernel = CuFunction(md, "normal_pdf_float")
    global block_sum_kernel = CuFunction(md, "block_sum_float")
    global sum_kernel = CuFunction(md, "sum_simplistic_float")
=#
  else
    throw("Invalid type specified")
  end
  return md
end

function unload_functions!(md::CuModule)
  # finalize: unload module
  unload(md)
end

function close_cuda(gpuid::Integer = first(devices(dev->capability(dev)[1]>=2)) )
  device_reset(gpuid)
end

#  Could optimize the choice of block sizes better, allow for 2-d blocks, etc.
function choose_block_size(n::Integer)
  min(n,max_threads_per_block)
end

#  Could optimize the chocie of grid sizes better, allow for 3-d grids, etc.
function choose_grid_size(n::Integer, block_size::Integer)
  @assert(block_size <= max_block_dimx_size)

  grid_size = ceil(Int64,n/block_size)
  if grid_size > max_grid_dimx_size
    grid_size_x = ceil(Int64,sqrt(grid_size))
    @assert(grid_size_x <= max_grid_dimx_size)
    grid_size = (grid_size_x, grid_size_x)
  end
  return grid_size
end

nothing


text:
using Random
using DataStructures
using Statistics
using Colors
using Images

"""
Example Use:
> rng = MersenneTwister(0)
> image = render(generatescene_objects(rng))
> save("scene.png", colorview(RGBA, image))
> println(parsescene_image(image))

Example Use for Inference
> rng = MersenneTwister(0)
> generate_and_save_random_scene_inf(rng) # saves image as scene2.png
"""

"""Generate and save random scene as scene.png"""
function generate_and_save_random_scene(rng)
  image = render(generatescene_objects(rng))
  save("scene.png", colorview(RGBA, image))
end

"""Generate and save random scene as scene_inf.png"""
function generate_and_save_random_scene_inf(rng)
  image = render_inf(generatescene_objects_inf(rng))
  save("scene_inf.png", colorview(RGBA, image))
end

# ----- NEW: render functions for inference ----- #

"""Input: list of objects, where each object is a tuple (shape, color, position)"""
function render_inf(objects; gridsize=16, transparent=false)
  background = "white"
  image = [RGBA(1.0, 0.0, 0.0, 1.0) for x in 1:gridsize, y in 1:gridsize]
  for object in objects
    center_x = object[3][1]
    center_y = object[3][2]
    shape = object[1]
    color = rgb(object[2])
    for shape_position in shape
      shape_x, shape_y = shape_position
      x = center_x + shape_x
      y = center_y + shape_y
      if (x > 0) && (x <= gridsize) && (y > 0) && (y <= gridsize) # only render in-bound pixel positions
        if transparent
          if image[y, x] == RGBA(1.0, 0.0, 0.0, 1.0)
            image[y, x] = RGBA(color.r, color.g, color.b, 0.6)
          else
            new_alpha = image[y,x].alpha + 0.6 - image[y,x].alpha * 0.6
            image[y, x] = RGBA((image[y,x].alpha * image[y,x].r + 0.6*(1 - image[y,x].alpha)*color.r)/new_alpha,
                               (image[y,x].alpha * image[y,x].g + 0.6*(1 - image[y,x].alpha)*color.g)/new_alpha,
                               (image[y,x].alpha * image[y,x].b + 0.6*(1 - image[y,x].alpha)*color.b)/new_alpha,
                              new_alpha)
          end
        else
          image[y, x] = RGBA(color.r, color.g, color.b, 0.6)
        end
      end
    end
  end
  for x in 1:gridsize
    for y in 1:gridsize
      if image[x, y] == RGBA(1.0, 0.0, 0.0, 1.0)
        image[x, y] = rgb(background)
      end
    end
  end
  image
end

function generatescene_objects_inf(rng=Random.GLOBAL_RNG; gridsize::Int=16)
  types, objects, background, gridsize = generatescene_objects(rng, gridsize=gridsize)
  formatted_objects = []
  for object in sort(objects, by=x->x.type.id)
    push!(formatted_objects, (object.type.shape, object.type.color, object.position))
  end
  formatted_objects
end

# ----- define colors and color-related functions ----- #

colors = ["red", "yellow", "green", "blue"]
backgroundcolors = ["white", "black"]

"""Euclidean distance between two RGB/RGBA colors"""
function colordist(color1, color2)
  (color1.r - color2.r)^2 + (color1.g - color2.g)^2 + (color1.b - color2.b)^2
end

"""CSS string color name from RGB color"""
function colorname(r::RGB)
  rgbs = vcat(keys(rgb_to_colorname)...)
  colordists = map(x -> colordist(r, x), rgbs)
  minidx = findall(x -> x == minimum(colordists), colordists)[1]
  rgb_key = rgbs[minidx]
  rgb_to_colorname[rgb_key]
end

"""CSS string color name from RGBA color"""
function colorname(rgba::RGBA)
  rgb = RGB(rgba.r, rgba.g, rgba.b)
  colorname(rgb)
end

"""RGB value from CSS string color name"""
function rgb(colorname)
  colorname_to_rgb[colorname]
end

rgb_to_colorname = Dict([
  (colorant"red", "red"),
  (colorant"yellow", "yellow"),
  (colorant"green", "green"),
  (colorant"blue", "blue"),
  (colorant"white", "white"),
  (colorant"black", "black")
]);

colorname_to_rgb = Dict([
  ("red", colorant"red"),
  ("yellow", colorant"yellow"),
  ("green", colorant"green"),
  ("blue", colorant"blue"),
  ("white", colorant"white"),
  ("black", colorant"black")
])

# ----- end define colors and color-related functions ----- #

# ----- define general utils ----- #

"""Compute neighbor positions of given shape"""
function neighbors(shape::AbstractArray)
  neighborPositions = vcat(map(pos -> neighbors(pos), shape)...)
  unique(filter(pos -> !(pos in shape), neighborPositions))
end

"""Compute neighbor positions of given position"""
function neighbors(position)
  x = position[1]
  y = position[2]
  [(x + 1, y), (x - 1, y), (x, y + 1), (x, y - 1)]
end

# ----- end define general utils ----- #

# ----- define functions related to generative model over scenes ----- #
mutable struct ObjType
  shape::AbstractArray
  color::String
  custom_fields::AbstractArray
  id::Int
end

mutable struct Obj
  type::ObjType
  position::Tuple{Int, Int}
  custom_field_values::AbstractArray
  id::Int
end

"""Produce image from types_and_objects scene representation"""
function render(types_and_objects)
  types, objects, background, gridsize = types_and_objects
  image = [RGBA(1.0, 0.0, 0.0, 1.0) for x in 1:gridsize, y in 1:gridsize]
  println("""
  (program
    (= GRID_SIZE $(gridsize))
    (= background "$(background)")
    $(join(map(t -> "(object ObjType$(t.id) (list $(join(map(cell -> """(Cell $(cell[1]) $(cell[2]) "$(t.color)")""", t.shape), " "))))", types), "\n  "))

    $((join(map(obj -> """(: obj$(obj.id) ObjType$(obj.type.id))""", objects), "\n  "))...)

    $((join(map(obj -> """(= obj$(obj.id) (initnext (ObjType$(obj.type.id) (Position $(obj.position[1] - 1) $(obj.position[2] - 1))) (prev obj$(obj.id))))""", objects), "\n  ")))
  )
  """)
  for object in objects
    center_x = object.position[1]
    center_y = object.position[2]
    type = object.type
    color = rgb(object.type.color)
    for shape_position in type.shape
      shape_x, shape_y = shape_position
      x = center_x + shape_x
      y = center_y + shape_y
      if (x > 0) && (x <= gridsize) && (y > 0) && (y <= gridsize) # only render in-bound pixel positions
        if image[y, x] == RGBA(1.0, 0.0, 0.0, 1.0)
          image[y, x] = RGBA(color.r, color.g, color.b, 0.6)
        else
          new_alpha = image[y,x].alpha + 0.6 - image[y,x].alpha * 0.6
          image[y, x] = RGBA((image[y,x].alpha * image[y,x].r + 0.6*(1 - image[y,x].alpha)*color.r)/new_alpha,
                             (image[y,x].alpha * image[y,x].g + 0.6*(1 - image[y,x].alpha)*color.g)/new_alpha,
                             (image[y,x].alpha * image[y,x].b + 0.6*(1 - image[y,x].alpha)*color.b)/new_alpha,
                            new_alpha)
        end
      end
    end
  end
  for x in 1:gridsize
    for y in 1:gridsize
      if image[x, y] == RGBA(1.0, 0.0, 0.0, 1.0)
        image[x, y] = rgb(background)
      end
    end
  end
  image
end

function program_string(types_and_objects)
  types, objects, background, gridsize = types_and_objects
  """
  (program
    (= GRID_SIZE $(gridsize))
    (= background "$(background)")
    $(join(map(t -> "(object ObjType$(t.id) (list $(join(map(cell -> """(Cell $(cell[1]) $(cell[2]) "$(t.color)")""", t.shape), " "))))", types), "\n  "))

    $((join(map(obj -> """(: obj$(obj.id) ObjType$(obj.type.id))""", objects), "\n  "))...)

    $((join(map(obj -> """(= obj$(obj.id) (initnext (ObjType$(obj.type.id) (Position $(obj.position[1] - 1) $(obj.position[2] - 1))) (prev obj$(obj.id))))""", objects), "\n  ")))
  )
  """
end

function program_string_synth(types_and_objects)
  types, objects, background, gridsize = types_and_objects
  """
  (program
    (= GRID_SIZE $(gridsize))
    (= background "$(background)")
    $(join(map(t -> "(object ObjType$(t.id) $(join(map(tuple -> "(: $(tuple[1]) $(tuple[2]))", t.custom_fields), " ")) (list $(join(map(cell -> """(Cell $(cell[1]) $(cell[2]) $(t.custom_fields == [] ? """ "$(t.color)" """ : "color"))""", t.shape), " "))))", types), "\n  "))

    $((join(map(obj -> """(: obj$(obj.id) ObjType$(obj.type.id))""", objects), "\n  "))...)

    $((join(map(obj -> """(= obj$(obj.id) (initnext (ObjType$(obj.type.id) $(join(map(v -> """ "$(v)" """, obj.custom_field_values), " ")) (Position $(obj.position[1]) $(obj.position[2]))) (prev obj$(obj.id))))""", objects), "\n  ")))
  )
  """
end

function program_string_synth_update_rule(types_and_objects)
  types, objects, background, gridsize = types_and_objects
  """
  (program
    (= GRID_SIZE $(gridsize))
    (= background "$(background)")
    $(join(map(t -> "(object ObjType$(t.id) $(join(map(tuple -> "(: $(tuple[1]) $(tuple[2]))", t.custom_fields), " ")) (list $(join(map(cell -> """(Cell $(cell[1]) $(cell[2]) $(t.custom_fields == [] ? """ "$(t.color)" """ : "color"))""", t.shape), " "))))", types), "\n  "))

    $((join(map(obj -> """(: obj$(obj.id) ObjType$(obj.type.id))""", objects), "\n  "))...)

    $((join(map(obj -> obj.position != (-1, -1) ? """(= obj$(obj.id) (initnext (ObjType$(obj.type.id) $(join(map(v -> """ "$(v)" """, obj.custom_field_values), " ")) (Position $(obj.position[1]) $(obj.position[2]))) (prev obj$(obj.id))))"""
                                                : """(= obj$(obj.id) (initnext (removeObj (ObjType$(obj.type.id) $(join(map(v -> """ "$(v)" """, obj.custom_field_values), " ")) (Position $(obj.position[1]) $(obj.position[2])))) (prev obj$(obj.id))))""", objects), "\n  ")))
  )
  """
end

function program_string_synth_standard_groups(object_decomposition)
  object_types, object_mapping, background, gridsize = object_decomposition

  start_objects = sort(filter(obj -> obj != nothing, [object_mapping[i][1] for i in 1:length(collect(keys(object_mapping)))]), by=(x -> x.id))

  start_type_mapping = Dict()
  num_objects_with_type = Dict()

  for type in object_types
    start_type_mapping[type.id] = sort(filter(obj -> obj.type.id == type.id, start_objects), by=(x -> x.id))
    num_objects_with_type[type.id] = count(obj_id -> filter(x -> !isnothing(x), object_mapping[obj_id])[1].type.id == type.id, collect(keys(object_mapping)))
  end

  start_constants_and_lists = vcat(filter(l -> length(l) == 1 && (num_objects_with_type[l[1].type.id] == 1), map(k -> start_type_mapping[k], collect(keys(start_type_mapping))))...,
                                   filter(l -> length(l) > 1 || ((length(l) == 1) && (num_objects_with_type[l[1].type.id] > 1)), map(k -> start_type_mapping[k], collect(keys(start_type_mapping)))))
  start_constants_and_lists = sort(start_constants_and_lists, by=x -> x isa Array ? x[1].id : x.id)

  other_list_types = filter(k -> (length(start_type_mapping[k]) == 0) || (length(start_type_mapping[k]) == 1 && num_objects_with_type[k] == 1), sort(collect(keys(start_type_mapping))))

  """
  (program
    (= GRID_SIZE $(gridsize))
    (= background "$(background)")
    $(join(map(t -> "(object ObjType$(t.id) $(join(map(tuple -> "(: $(tuple[1]) $(tuple[2]))", t.custom_fields), " ")) (list $(join(map(cell -> """(Cell $(cell[1]) $(cell[2]) $(t.custom_fields == [] ? """ "$(t.color)" """ : "color"))""", t.shape), " "))))", object_types), "\n  "))

    $((join(map(obj -> obj isa Array ? """(: addedObjType$(obj[1].type.id)List (List ObjType$(obj[1].type.id)))"""
                                     : """(: obj$(obj.id) ObjType$(obj.type.id))""", start_constants_and_lists), "\n  "))...)

    $((join(map(k -> """(: addedObjType$(k)List (List ObjType$(k)))""", other_list_types), "\n  "))...)

    $((join(map(obj -> obj isa Array ? """(= addedObjType$(obj[1].type.id)List (initnext (list $(join(map(x -> "(ObjType$(x.type.id) $(join(map(v -> v isa String ? """ "$(v)" """ : "$(v)", x.custom_field_values), " ")) (Position $(x.position[1]) $(x.position[2])))", obj), " "))) (prev addedObjType$(obj[1].type.id)List)))"""
                                     : """(= obj$(obj.id) (initnext (ObjType$(obj.type.id) $(join(map(v -> v isa String ? """ "$(v)" """ : "$(v)", obj.custom_field_values), " ")) (Position $(obj.position[1]) $(obj.position[2]))) (prev obj$(obj.id))))""", start_constants_and_lists), "\n  ")))

    $(join(map(k -> """(= addedObjType$(k)List (initnext (list) (prev addedObjType$(k)List)))""", other_list_types), "\n  ")...)
  )
  """
end

function program_string_synth_custom_groups(object_decomposition, group_structure)
  object_types, object_mapping, background, gridsize = object_decomposition

  objects = group_structure["objects"] # list of single-variable objects (ordered by id)
  group_dict = group_structure["groups"] # dictionary from group ids to list of ordered object ids
  group_ids = sort(collect(keys(group_dict)))

  """
  (= GRID_SIZE $(gridsize))
  (= background "$(background)")
  $(join(map(t -> "(object ObjType$(t.id) $(join(map(tuple -> "(: $(tuple[1]) $(tuple[2]))", t.custom_fields), " ")) (list $(join(map(cell -> """(Cell $(cell[1]) $(cell[2]) $(t.custom_fields == [] ? """ "$(t.color)" """ : "color"))""", t.shape), " "))))", object_types), "\n  "))

  $(join(map(obj -> """(: obj$(obj.id) ObjType$(obj.type.id))""", objects), "\n  "))
  $(join(map(group_id -> """(: group$(group_id) (List ObjType$(filter(x -> !isnothing(x), object_mapping[group_dict[group_id][1]])[1].type.id)))""", group_ids), "\n  "))

  $(join(map(obj -> """(= obj$(obj.id) (initnext (ObjType$(obj.type.id) $(join(map(v -> """ "$(v)" """, obj.custom_field_values), " ")) (Position $(obj.position[1]) $(obj.position[2]))) (prev obj$(obj.id))))""", objects), "\n  "))
  $(join(map(group_id -> """(= group$(group_id) (initnext (list $(join(map(obj -> """(ObjType$(obj.type.id) $(join(map(v -> """ "$(v)" """, obj.custom_field_values), " ")) (Position $(obj.position[1]) $(obj.position[2])))""" , filter(x -> !isnothing(x), map(obj_id -> object_mapping[obj_id][1], group_dict[group_id]))), " "))) (prev group$(group_id))))""", group_ids), "\n  "))
  """
end

function generatescene_program(rng=Random.GLOBAL_RNG; gridsize::Int=16)
  types_and_objects = generatescene_objects(rng, gridsize=gridsize)
  program_string(types_and_objects)
end

function generatescene_objects(rng=Random.GLOBAL_RNG; gridsize::Int=16)
  background = backgroundcolors[rand(1:length(backgroundcolors))]
  numObjects = rand(rng, 1:20)
  numTypes = rand(rng, 1:min(numObjects, 5))
  types = [] # each type has form (list of position tuples, index in types list)::Tuple{Array{Tuple{Int, Int}}, Int}

  objectPositions = [(rand(rng, 1:gridsize), rand(rng, 1:gridsize)) for x in 1:numObjects]
  objects = [] # each object has form (type, position tuple, color, index in objects list)

  for type in 1:numTypes
    renderSize = rand(rng, 1:5)
    shape = [(0,0)]
    while length(shape) != renderSize
      boundaryPositions = neighbors(shape)
      push!(shape, boundaryPositions[rand(rng, 1:length(boundaryPositions))])
    end
    color = colors[rand(rng, 1:length(colors))]

    # generate custom fields
    custom_fields = []
    num_fields = rand(0:2)
    for i in 1:num_fields
      push!(custom_fields, ("field$(i)", rand(["Int", "Bool"])))
    end

    push!(types, ObjType(shape, color, custom_fields, length(types) + 1))
  end

  for i in 1:numObjects
    objPosition = objectPositions[i]
    objType = types[rand(rng, 1:length(types))]

    # generate custom field values
    custom_fields = objType.custom_fields
    custom_field_values = map(field -> field[2] == "Int" ? rand(1:3) : rand(["true", "false"]), custom_fields)

    push!(objects, Obj(objType, objPosition, custom_field_values, length(objects) + 1))
  end
  (types, objects, background, gridsize)
end

# ----- end functions related to generative model over scenes ----- #

# ----- define functions related to scene parsing -----

function parsescene_image_singlecell(image)
  dimImage = size(image)[1]
  background = count(x -> x == "white", map(colorname, image)) > count(x -> x == "black", map(colorname, image)) ? "white" : "black"
  colors = []
  objects = []
  for y in 1:dimImage
    for x in 1:dimImage
      color = colorname(image[y, x])
      if color != "white"
        if !(color in colors)
          push!(colors, color)
        end

        push!(objects, (x - 1, y - 1, (color, findall(x -> x == color, colors)[1]), length(objects) + 1))
      end
    end
  end

  """
  (program
    (= GRID_SIZE $(dimImage))
    (= background "$(background)")

    $(join(map(color -> """(object ObjType$(findall(x -> x == color, colors)[1]) (Cell 0 0 "$(color)"))""", colors), "\n  "))

    $(join(map(obj -> """(: obj$(obj[4]) ObjType$(obj[3][2]))""", objects), "\n  "))

    $(join(map(obj -> """(= obj$(obj[4]) (initnext (ObjType$(obj[3][2]) (Position $(obj[1]) $(obj[2]))) (prev obj$(obj[4]))))""", objects), "\n  "))
  )
  """
end

function color_contiguity(image, pos1, pos2)
  image[pos1[1], pos1[2]] == image[pos2[1], pos2[2]]
end

function parsescene_image(image; color=true)
  dimImage = size(image)[1]
  background = count(x -> x == "white", map(colorname, image)) > count(x -> x == "black", map(colorname, image)) ? "white" : "black"
  objectshapes = []
  colored_positions = map(ci -> (ci.I[2], ci.I[1]), findall(color -> color != "white", map(colorname, image)))
  visited = []
  for position in colored_positions
    if !(position in visited)
      objectshape = []
      q = Queue{Any}()
      enqueue!(q, position)
      while !isempty(q)
        pos = dequeue!(q)
        push!(objectshape, pos)
        push!(visited, pos)
        pos_neighbors = neighbors(pos)
        for n in pos_neighbors
          if (n in colored_positions) && !(n in visited) && (color ? color_contiguity(image, n, pos) : true)
            enqueue!(q, n)
          end
        end
      end
      push!(objectshapes, objectshape)
    end
  end

  types = []
  objects = []
  for objectshape in objectshapes
    objectcolors = map(pos -> colorname(image[pos[2], pos[1]]), objectshape)

    translated = map(pos -> dimImage * (pos[2] - 1)+ (pos[1] - 1), objectshape)
    translated = length(translated) % 2 == 0 ? translated[1:end-1] : translated
    centerPos = objectshape[findall(x -> x == median(translated), translated)[1]]
    translatedShape = map(pos -> (pos[1] - centerPos[1], pos[2] - centerPos[2]), objectshape)
    translatedShapeWithColors = [(translatedShape[i], objectcolors[i]) for i in 1:length(translatedShape)]

    push!(types, (translatedShapeWithColors, length(types) + 1))
    push!(objects, (centerPos, length(types), length(objects) + 1))
  end

  """
  (program
    (= GRID_SIZE $(dimImage))
    (= background "$(background)")

    $(join(map(t -> """(object ObjType$(t[2]) (list $(join(map(cell -> """(Cell $(cell[1][1]) $(cell[1][2]) "$(cell[2])")""", t[1]), " ")))""", types), "\n  "))

    $(join(map(obj -> """(: obj$(obj[3]) ObjType$(obj[2]))""", objects), "\n  "))

    $(join(map(obj -> """(= obj$(obj[3]) (initnext (ObjType$(obj[2]) (Position $(obj[1][1] - 1) $(obj[1][2] - 1))) (prev obj$(obj[3]))))""", objects), "\n  "))
  )
  """
end

function color_contiguity_autumn(position_to_color, pos1, pos2)
  length(intersect(position_to_color[pos1], position_to_color[pos2])) > 0
end

function parsescene_autumn(render_output::AbstractArray, dim::Int=16, background::String="white"; color=true)
  colors = unique(map(cell -> cell.color, render_output))

  objectshapes = []
  for c in colors
    colored_positions = sort(map(cell -> (cell.position.x, cell.position.y), filter(cell -> cell.color == c, render_output)))
    if length(unique(colored_positions)) == length(colored_positions) # no overlapping objects of the same color
      visited = []
      for position in colored_positions
        if !(position in visited)
          objectshape = []
          q = Queue{Any}()
          enqueue!(q, position)
          while !isempty(q)
            pos = dequeue!(q)
            push!(objectshape, (pos, c))
            push!(visited, pos)
            pos_neighbors = neighbors(pos)
            for n in pos_neighbors
              if (n in colored_positions) && !(n in visited)
                enqueue!(q, n)
              end
            end
          end
          push!(objectshapes, objectshape)
        end
      end
    else # overlapping objects of the same color; parse each cell as a different object
      for position in colored_positions
        push!(objectshapes, [(position, c)])
      end
    end
  end
  @show objectshapes

  types = []
  objects = []
  # @show length(objectshapes)
  for objectshape_with_color in objectshapes
    objectcolor = objectshape_with_color[1][2]
    objectshape = map(o -> o[1], objectshape_with_color)
    # @show objectcolor
    # @show objectshape
    translated = map(pos -> dim * pos[2] + pos[1], objectshape)
    translated = length(translated) % 2 == 0 ? translated[1:end-1] : translated # to produce a single median
    centerPos = objectshape[findall(x -> x == median(translated), translated)[1]]
    translatedShape = unique(map(pos -> (pos[1] - centerPos[1], pos[2] - centerPos[2]), sort(objectshape)))

    if !((translatedShape, objectcolor) in map(type -> (type.shape, type.color) , types))
      push!(types, ObjType(translatedShape, objectcolor, [], length(types) + 1))
      push!(objects, Obj(types[length(types)], centerPos, [], length(objects) + 1))
    else
      type_id = findall(type -> (type.shape, type.color) == (translatedShape, objectcolor), types)[1]
      push!(objects, Obj(types[type_id], centerPos, [], length(objects) + 1))
    end
  end

  # combine types with the same shape but different colors
  println("INSIDE REGULAR PARSER")
  @show types
  @show objects
  (types, objects) = combine_types_with_same_shape(types, objects)

  (types, objects, background, dim)
end

function parsescene_autumn_given_types(render_output::AbstractArray, override_types::AbstractArray, dim::Int=16, background::String="white"; color=true)
  (standard_types, objects, _, _) = parsescene_autumn(render_output, dim, background, color=color)
  println("OBJECTS")
  println(objects)
  println("OBJECT_TYPES")
  println(standard_types)
  # extract multi-cellular types that do not appear in override_types
  old_override_types = deepcopy(override_types)
  @show override_types
  @show standard_types
  types_to_ungroup = filter(s_type -> (length(s_type.shape) > 1), standard_types)

  # extract single-cell types
  grouped_type_colors = map(grouped_type -> length(grouped_type.custom_fields) == 0 ? [grouped_type.color] : grouped_type.custom_fields[1][3], types_to_ungroup)
  composition_types = map(colors -> (filter(type -> (length(type.shape) == 1) && (( (length(type.custom_fields) == 0) && (length(intersect(colors, [type.color])) > 0))
                                                                                || (length(type.custom_fields) > 0) && (length(intersect(colors, type.custom_fields[1][3])) > 0)), standard_types),
                                     filter(type -> (length(type.shape) == 1) && (( (length(type.custom_fields) == 0) && (length(intersect(colors, [type.color])) > 0))
                                                                                || (length(type.custom_fields) > 0) && (length(intersect(colors, type.custom_fields[1][3])) > 0)), override_types)),
                          grouped_type_colors)

  # only consider types to ungroup that have single-celled types of the same color
  @show types_to_ungroup
  @show composition_types
  @show length(types_to_ungroup)
  @show length(composition_types)
  remove_types_to_ungroup = []
  for i in 1:length(composition_types)
    if composition_types[i] == ([], [])
      println("WHAT")
      println(map(type -> type.id, types_to_ungroup))
      println(types_to_ungroup[i].id)
      push!(remove_types_to_ungroup, types_to_ungroup[i])
      println(length(types_to_ungroup))
    end
  end
  types_to_ungroup = filter(t -> !(t.id in map(type -> type.id, remove_types_to_ungroup)), types_to_ungroup)
  composition_types = filter(types -> types != ([], []), composition_types)

  println("READY")
  @show types_to_ungroup
  @show composition_types

  if (length(types_to_ungroup) == 0) # no types to try ungrouping
    new_objects = objects
    new_types = standard_types
  else # exist types to ungroup
    @show types_to_ungroup
    new_objects = filter(obj -> !(obj.type.id in map(type -> type.id, types_to_ungroup)), objects)
    new_types = standard_types
    # println("HELLO 1")
    # @show new_types
    for grouped_type_id in 1:length(types_to_ungroup)
      grouped_type = types_to_ungroup[grouped_type_id]
      composition_types_group = composition_types[grouped_type_id]

      if length(grouped_type.custom_fields) == 0
        filter!(type -> type.id != grouped_type.id, new_types) # remove grouped type from new_types
        println("-------------------> LOOK AT ME")
        # @show new_types
        # determine composition type
        @show grouped_type_id
        @show composition_types_group
        if length(composition_types_group[1]) > 0 # composition type present in standard types
          composition_type = composition_types_group[1][1]
        else # composition type present in override types only
          composition_type = composition_types_group[2][1]
          composition_type.id = grouped_type.id # switch the composition type's id to the grouped type's id, since we're eliminating the grouped type
          push!(new_types, composition_type)
        end

        objects_to_ungroup = filter(obj -> obj.type.id == grouped_type.id, objects)
        # @show objects_to_ungroup
        for object in objects_to_ungroup
          for pos in object.type.shape
            if composition_type.custom_fields == []
              push!(new_objects, Obj(composition_type, (pos[1] + object.position[1], pos[2] + object.position[2]), [], length(new_objects) + length(objects)))
            else
              push!(new_objects, Obj(composition_type, (pos[1] + object.position[1], pos[2] + object.position[2]), [object.type.color], length(new_objects) + length(objects)))
            end
          end
        end
      else # grouped object supports multiple colors
        println("HERE I AM 2")
        colors = deepcopy(grouped_type.custom_fields[1][3])
        println(colors)
        for color in colors
          println("HERE I AM 3")
          @show composition_types_group
          println(vcat(vcat(map(types_list -> map(type -> vcat(type.color, (length(type.custom_fields) == 0 ? [] : type.custom_fields[1][3])...), types_list), composition_types_group)...)...))
          if color in vcat(vcat(map(types_list -> map(type -> vcat(type.color, (length(type.custom_fields) == 0 ? [] : type.custom_fields[1][3])...), types_list), composition_types_group)...)...)
            println("HERE I AM")
            @show grouped_type
            @show color
            if length(composition_types_group[1]) > 0 # composition type present in standard types
              in_standard_bool = true
              type = composition_types_group[1][1]
              if color in vcat(type.color, (type.custom_fields == [] ? [] : type.custom_fields[1][3])...)
                composition_type = type
              end
            else # composition type present in override types only
              in_standard_bool = false
              type = composition_types_group[2][1]
              if color in vcat(type.color, (type.custom_fields == [] ? [] : type.custom_fields[1][3])...)
                composition_type = type
              end
            end

            # remove color from grouped type OR remove object if all colors have been removed
            if length(grouped_type.custom_fields[1][3]) == 2
              filter!(c -> c != color, grouped_type.custom_fields[1][3])
              type.color = color
              println("WHY 1")
              println(grouped_type)
            elseif length(grouped_type.custom_fields[1][3]) == 1
              filter!(type -> type.id != grouped_type.id, new_types) # remove object if all colors have been eliminated
            else
              filter!(c -> c != color, grouped_type.custom_fields[1][3])
            end
            println("-----> HERE 2")
            println(new_types)

            if !(in_standard_bool)
              composition_type.id = length(new_types) + length(override_types) + 1
              push!(new_types, composition_type)
              println("----> HERE")
              println(composition_type)
            end

            objects_to_ungroup = filter(obj -> (obj.type.id == grouped_type.id) &&
                                               (((obj.custom_field_values == []) && obj.color == color) || (obj.custom_field_values == [color])), objects)
            println("OBJECTS TO UNGROUP")
            println(grouped_type.id)
            println(objects_to_ungroup)
            for object in objects_to_ungroup
              for pos in object.type.shape
                if composition_type.custom_fields == []
                  push!(new_objects, Obj(composition_type, (pos[1] + object.position[1], pos[2] + object.position[2]), [], length(new_objects) + length(objects)))
                else
                  push!(new_objects, Obj(composition_type, (pos[1] + object.position[1], pos[2] + object.position[2]), object.custom_field_values != [] ? object.custom_field_values : [object.type.color], length(new_objects) + length(objects)))
                end
              end
            end

          else # color not in custom_fields
            println("ADD BACK?")
            println(objects)
            println(grouped_type.id)
            println(filter(obj -> (obj.type.id == grouped_type.id) && (obj.custom_field_values == [color]), objects))
            # add previously removed objects back
            push!(new_objects, filter(obj -> (obj.type.id == grouped_type.id) && (obj.custom_field_values == [color]), objects)...)
          end
        end
      end
    end
    # println("HELLO 3")
    # @show new_types
    # @show new_objects
    # re-number type id's
    sort!(new_types, by = x -> x.id)
    for i in 1:length(new_types)
      type = new_types[i]
      if type.id != i
        foreach(o -> o.type.id = i, filter(obj -> obj.type.id == type.id, new_objects))
        type.id = i
      end
    end

    # re-number object id's
    sort!(new_objects, by = x -> x.id)
    for i in 1:length(new_objects)
      object = new_objects[i]
      object.id = i
    end

    for type in new_types
      if (type.custom_fields != []) && (length(type.custom_fields[1][3]) == 1)
        type.color = type.custom_fields[1][3][1]
        type.custom_fields = []
      end
    end

    for object in new_objects
      if object.type.custom_fields == []
        object.custom_field_values = []
      end
    end
  end

  println("BEFORE COMBINING TYPES INTO ONE WITH COLOR FIELD")
  @show new_types
  @show new_objects

  # group objects with same shape but different colors into one type
  new_types, new_objects = combine_types_with_same_shape(new_types, new_objects)
  println("POST COMBINING")
  @show new_types
  @show new_objects

  # take the union of override_types and new_types
  @show old_override_types
  old_types = deepcopy(old_override_types)
  new_object_colors = map(o -> o.type.color, new_objects)
  types_to_add = []
  for new_type in new_types
    new_type_shape = new_type.shape
    if new_type_shape in map(t -> t.shape, old_types)
      override_type = old_types[findall(t -> t.shape == new_type_shape, old_types)[1]]
      if length(override_type.custom_fields) == 0
        if length(new_type.custom_fields) == 0
          if override_type.color != new_type.color
            push!(override_type.custom_fields, ("color", "String", [override_type.color, new_type.color]))
          end
        else
          push!(override_type.custom_fields, ("color", "String", unique([override_type.color, new_type.custom_fields[1][3]...])))
        end
      else
        if length(new_type.custom_fields) == 0
          colors = override_type.custom_fields[1][3]
          push!(colors, new_type.color)
          unique!(colors)
        else
          push!(override_type.custom_fields[1][3], new_type.custom_fields[1][3]...)
          unique!(override_type.custom_fields[1][3])
        end
      end
    else
      new_type.id = length(old_types) + length(types_to_add) + 1
      push!(types_to_add, new_type)
    end
  end
  new_types = vcat(old_types..., types_to_add...)

  for i in 1:length(new_types)
    new_types[i].id = i
  end

  println("POST UNION")
  println(new_types)

  # reassign objects
  for i in 1:length(new_objects)
    object = new_objects[i]
    # new type
    type = new_types[findall(t -> t.shape == object.type.shape, new_types)[1]]
    if type.custom_fields != [] && object.custom_field_values == []
      push!(object.custom_field_values, new_object_colors[i])
    end
    object.type = type
  end

  (new_types, new_objects, background, dim)

end

function combine_types_with_same_shape(object_types, objects)
  println("COMBINE TYPES WITH SAME SHAPE")
  println(object_types)
  println(objects)
  types_to_remove = []
  for i in 1:length(object_types)
    type_i = object_types[i]
    type_i_shape = type_i.shape
    for j in i:length(object_types)
      type_j = object_types[j]
      type_j_shape = object_types[j].shape
      if (i != j) && (type_i_shape == type_j_shape)
        push!(types_to_remove, type_j)
        if "color" in map(tuple -> tuple[1], type_i.custom_fields)
          colors = type_i.custom_fields[findall(tuple -> tuple[1] == "color", type_i.custom_fields)[1]][3]
          push!(colors, type_j.color)
          unique!(colors)
        else
          push!(type_i.custom_fields, ("color", "String", [type_i.color, type_j.color]))
          unique!(type_i.custom_fields)

          objects_to_update_type_i = filter(obj -> obj.type.id == type_i.id, objects)
          foreach(o -> push!(o.custom_field_values, o.type.color) , objects_to_update_type_i)
        end
        # objects_to_update_type_i = filter(obj -> obj.type.id == type_i.id, objects)
        # foreach(o -> push!(o.custom_field_values, o.type.color) , objects_to_update_type_i)

        objects_to_update_type_j = filter(obj -> obj.type.id == type_j.id, objects)
        foreach(o -> o.type = type_i, objects_to_update_type_j)
        foreach(o -> o.custom_field_values = o.custom_field_values == [] ? [type_j.color] : o.custom_field_values, objects_to_update_type_j)
      end
    end
  end
  object_types = filter(type -> !(type in types_to_remove), object_types)

  # re-number type id's
  sort!(object_types, by = x -> x.id)
  for i in 1:length(object_types)
    type = object_types[i]
    if type.id != i
      foreach(o -> o.type.id = i, filter(obj -> obj.type.id == type.id, objects))
      type.id = i
    end
  end

  # re-number object id's
  sort!(objects, by = x -> x.id)
  for i in 1:length(objects)
    object = objects[i]
    object.id = i
  end

  println("END COMBINE TYPES WITH SAME SHAPE")
  println(object_types)
  println(objects)
  (object_types, objects)
end

"""
mutable struct ObjType
  shape::AbstractArray
  color::String
  custom_fields::AbstractArray
  id::Int
end

mutable struct Obj
  type::ObjType
  position::Tuple{Int, Int}
  custom_field_values::AbstractArray
  id::Int
end
"""

function parsescene_autumn_singlecell(render_output::AbstractArray, background::String="white", dim::Int=16)
  colors = unique(map(cell -> cell.color, render_output))
  types = map(color -> ObjType([(0,0)], color, [], findall(c -> c == color, colors)[1]), colors)
  objects = []
  for i in 1:length(render_output)
    cell = render_output[i]
    push!(objects, Obj(types[findall(type -> type.color == cell.color, types)[1]], (cell.position.x, cell.position.y), [], i))
  end
  (types, objects, background, dim)
end

function parsescene_autumn_singlecell_given_types(render_output::AbstractArray, override_types::AbstractArray, background::String="white", dim::Int=16)
  standard_types, objects, _, _ = parsescene_autumn_singlecell(render_output, background, dim)
  println("STANDARD TYPES ")
  println(standard_types)
  standard_types = deepcopy(standard_types)
  override_types = deepcopy(override_types)
  # compute union of standard types and override_types
  new_types = filter(type -> !(type.color in map(t -> t.color, override_types)), standard_types)
  for i in 1:length(new_types)
    type = new_types[i]
    type.id = length(override_types) + i
  end
  println("RETURN VAL")
  println(vcat(override_types..., new_types...))
  final_types = vcat(override_types..., new_types...)
  # ensure objects have correct types
  for object in objects
    object.type = final_types[findall(t -> t.color == object.type.color, final_types)[1]]
  end

  (final_types, objects, background, dim)
end

# ----- end functions related to scene parsing ----- #


text:
using KernelAbstractions.Extras: @unroll

export HBFVReconstruction

using ..DGMethods.FVReconstructions: AbstractReconstruction
import ..DGMethods.FVReconstructions: width

struct HBFVReconstruction{M, R} <: AbstractReconstruction
    _atmo::M
    _recon::R
end

width(hb_recon::HBFVReconstruction) = width(hb_recon._recon)

function (hb_recon::HBFVReconstruction)(
    state_bot,
    state_top,
    cell_states::SVector{D},
    cell_weights,
) where {D}
    FT = eltype(state_bot)
    vars_prim = Vars{vars_state(hb_recon._atmo, Primitive(), FT)}
    param_set = parameter_set(hb_recon._atmo)
    _grav = FT(grav(param_set))

    # stencil info
    stencil_diameter = D
    stencil_width = div(D - 1, 2)
    stencil_center = stencil_width + 1
    # save the pressure Δpressure_ref states
    ps = similar(state_bot, Size(stencil_diameter))
    ρgΔz_half = similar(state_bot, Size(stencil_diameter))

    @inbounds begin
        @unroll for i in 1:stencil_diameter
            ps[i] = vars_prim(cell_states[i]).p
            ρgΔz_half[i] =
                vars_prim(cell_states[i]).ρ * _grav * cell_weights[i] / 2
        end

        # construct reference pressure states and update pressure states
        p_ref = ps[stencil_center]
        p_bot_ref = p_ref + ρgΔz_half[stencil_center]
        p_top_ref = p_ref - ρgΔz_half[stencil_center]

        vars_prim(cell_states[stencil_center]).p -= p_ref

        p⁺_ref, p⁻_ref = p_ref, p_ref
        @unroll for i in 1:stencil_width
            # stencil_center - i , stencil_center - i + 1
            p⁻_ref +=
                ρgΔz_half[stencil_center - i + 1] +
                ρgΔz_half[stencil_center - i]
            vars_prim(cell_states[stencil_center - i]).p -= p⁻_ref

            # stencil_center + i - 1 , stencil_center + i
            p⁺_ref -=
                ρgΔz_half[stencil_center + i - 1] +
                ρgΔz_half[stencil_center + i]
            vars_prim(cell_states[stencil_center + i]).p -= p⁺_ref
        end

        hb_recon._recon(state_bot, state_top, cell_states, cell_weights)

        vars_prim(state_bot).p += p_bot_ref
        vars_prim(state_top).p += p_top_ref

        # reverse the pressure states back
        @unroll for i in 1:stencil_diameter
            vars_prim(cell_states[i]).p = ps[i]
        end
    end
end


text:
# scheduler metadata store
# -------------------------
# Used to share metadata among processes in a node (result and refcount).
# It would be possible to switch between different implementations of ExecutorMeta.

abstract type ExecutorMeta end

"""
ShareMode holds sharing statistics and determines share-first/help-first mode.
"""
mutable struct ShareMode
    nshared::Int
    ncreated::Int
    ndeleted::Int
    sharesnapshot::Tuple{Int,Int}
    sharethreshold::Int
    shouldshare::Bool

    function ShareMode(sharethreshold::Int)
        new(0, 0, 0, (0,0), sharethreshold, true)
    end
end

function take_share_snapshot(sm::ShareMode)
    t1 = sm.sharesnapshot
    t2 = (sm.ncreated, sm.ndeleted)
    sm.sharesnapshot = t2
    incr = t2 .- t1
    incr[1] - incr[2] # residual tasks shared in the interval (>0 => stop sharing more)
end

function should_share(sm::ShareMode)
    if sm.nshared == 0
        (sm.sharethreshold > 0)
    elseif sm.nshared <= sm.sharethreshold
        incr = take_share_snapshot(sm)
        sm.shouldshare = (incr == 0) ? sm.shouldshare : (incr < 0)
    else
        false
    end
end

function should_share(sm::ShareMode, nreserved::Int)
    (nreserved > sm.sharethreshold) && (sm.nshared == 0)
end

function reset(sm::ShareMode)
    sm.nshared = sm.ncreated = sm.ndeleted = 0
    sm.sharesnapshot = (0, 0)
    sm.shouldshare = true
    nothing
end

struct Packed
    res
end

#meta_deser(v) = meta_deser(String(v))
#meta_deser(v::String) = meta_deser(base64decode(v))
meta_deser(v::Vector{UInt8}) = deserialize(IOBuffer(v))
function meta_ser(x)
    iob = IOBuffer()
    serialize(iob, x)
    #base64encode(take!(iob))
    take!(iob)
end

meta_pack(v::Chunk) = isa(v.handle, DRef) ? chunktodisk(v) : v
meta_pack(v::Vector{Chunk}) = Packed(chunktodisk(Dagger.tochunk(chunktodisk(v))))
meta_pack(v) = Packed(chunktodisk(Dagger.tochunk(v)))
meta_unpack(v) = v
meta_unpack(v::Packed) = collect(v.res)

function repurpose_result_to_export(t::Thunk, val)
    if !isa(val, Chunk)
        val = Dagger.tochunk(val, persist = t.persist, cache = t.persist ? true : t.cache)
    end
    if isa(val.handle, DRef)
        val = chunktodisk(val)
    end
    val
end

function brokercall(fn, M, args...)
    result = remotecall_fetch(fn, M.brokerid, args...)
    if isa(result, Exception)
        @show result
        throw(result)
    end
    result
end

default_task_selector(x) = 1

resultroot{T<:ExecutorMeta}(M::T) = joinpath(M.path, "R")
resultpath{T<:ExecutorMeta}(M::T, id::TaskIdType) = joinpath(resultroot(M), string(id))
sharepath{T<:ExecutorMeta}(M::T, id::TaskIdType) = joinpath(M.path, "S", string(id))
taskpath{T<:ExecutorMeta}(M::T) = joinpath(M.path, "B", string(M.brokerid))

should_share{T<:ExecutorMeta}(M::T) = should_share(M.sharemode)
should_share{T<:ExecutorMeta}(M::T, nreserved::Int) = should_share(M.sharemode, nreserved)

init{T<:ExecutorMeta}(M::T, brokerid::Int; add_annotation=identity, del_annotation=identity, result_callback=nothing) = error("method not implemented for $T")
wait_trigger{T<:ExecutorMeta}(M::T; timeoutsec::Int=5) = error("method not implemented for $T")
delete!{T<:ExecutorMeta}(M::T) = error("method not implemented for $T")
reset{T<:ExecutorMeta}(M::T) = error("method not implemented for $T")
cleanup{T<:ExecutorMeta}(M::T) = error("method not implemented for $T")
detach{T<:ExecutorMeta}(M::T, pid) = error("method not implemented for $T")
share_task{T<:ExecutorMeta}(M::T, id::TaskIdType, allow_dup::Bool) = error("method not implemented for $T")
steal_task{T<:ExecutorMeta}(M::T, selector=default_task_selector) = error("method not implemented for $T")
set_result{T<:ExecutorMeta}(M::T, id::TaskIdType, val; refcount::Int=0, processlocal::Bool=true) = error("method not implemented for $T")
get_result{T<:ExecutorMeta}(M::T, id::TaskIdType) = error("method not implemented for $T")
has_result{T<:ExecutorMeta}(M::T, id::TaskIdType) = error("method not implemented for $T")
decr_result_ref{T<:ExecutorMeta}(M::T, id::TaskIdType) = error("method not implemented for $T")
export_local_result{T<:ExecutorMeta}(M::T, id::TaskIdType, executable) = error("method not implemented for $T")

function get_type(s::String)
    T = Main
    for t in split(s, ".")
        T = eval(T, Symbol(t))
    end
    T
end

metastore(name::String, args...) = (get_type(name))(args...)

# include the meta implementations

# ShmemMeta - uses shared memory as metadata store
module ShmemMeta

using Semaphores
using SharedDataStructures

import ..DagScheduler
import ..DagScheduler: TaskIdType, ExecutorMeta, ShareMode, NoTask, BcastChannel,
        take_share_snapshot, should_share, reset, cleanup, detach, meta_deser, meta_ser, meta_unpack, meta_pack, resultroot, resultpath, sharepath, taskpath,
        init, delete!, wait_trigger, share_task, steal_task, set_result, get_result, has_result, decr_result_ref,
        export_local_result, repurpose_result_to_export, register, deregister, put!, brokercall, default_task_selector, @timetrack, statetrack, logmsg

export ShmemExecutorMeta

const pinger = BcastChannel{Void}()

include("shmem_meta_store.jl")

end # module ShmemMeta

# SimpleMeta - uses Julia messaging and remotecalls
module SimpleMeta

using Base.Threads

import ..DagScheduler
import ..DagScheduler: TaskIdType, ExecutorMeta, ShareMode, NoTask, BcastChannel,
        take_share_snapshot, should_share, reset, cleanup, detach, meta_deser, meta_ser, meta_unpack, meta_pack, resultroot, resultpath, sharepath, taskpath,
        init, delete!, wait_trigger, share_task, steal_task, set_result, get_result, has_result, decr_result_ref,
        export_local_result, repurpose_result_to_export, register, deregister, put!, brokercall, default_task_selector, @timetrack, statetrack, logmsg

export SimpleExecutorMeta

const Results = BcastChannel{Tuple{String,Vector{UInt8}}}

const META = Dict{String,String}()
const TASKS = Vector{TaskIdType}()
const RESULTS = Results()
const PID_RR_MAP = Dict{Int,RemoteChannel}()
const PID_TASK_MAP = Dict{Int, Dict{TaskIdType,TaskIdType}}()
const taskmutex = Channel{Bool}(1)

include("simple_meta_store.jl")

function __init__()
    put!(taskmutex, true)
    nothing
end

function withtaskmutex(f)
    l = take!(taskmutex)
    try
        return f()
    finally
        put!(taskmutex, l)
    end
end

end # module SimpleMeta

# FdbMeta - uses foundationdb
module FdbMeta

using FoundationDB

import ..DagScheduler
import ..DagScheduler: TaskIdType, ExecutorMeta, ShareMode, NoTask, BcastChannel,
        take_share_snapshot, should_share, reset, cleanup, detach, meta_deser, meta_ser, meta_unpack, meta_pack, resultroot, resultpath, sharepath, taskpath,
        init, delete!, wait_trigger, share_task, steal_task, set_result, get_result, has_result, decr_result_ref,
        export_local_result, repurpose_result_to_export, register, deregister, put!, brokercall, default_task_selector, @timetrack, statetrack, logmsg

export FdbExecutorMeta

include("fdb_queue.jl")
include("fdb_dict.jl")
include("fdb_meta_store.jl")

const fdb_cluster = Ref{Union{Void,FDBCluster}}(nothing)
const fdb_db = Ref{Union{Void,FDBDatabase}}(nothing)

function __init__()
    try
        start_client()

        fdb_cluster[] = open(FDBCluster())
        fdb_db[] = open(FDBDatabase(fdb_cluster[]))

        atexit() do
            close(fdb_db[])
            close(fdb_cluster[])
            stop_client()
        end
    catch ex
        info("FdbMeta can not be used. FoundationDB could not be initialized.")
    end
    nothing
end

end # module FdbMeta


text:
lines = readlines(ARGS[1])

"""
): 3 points.
]: 57 points.
}: 1197 points.
>: 25137 points.
"""

function get_syntax_score(token)
    if token == ')'
        return 3
    elseif token == ']'
        return 57
    elseif token == '}'
        return 1197
    elseif token == '>'
        return 25137
    else
        return nothing
    end
end

function matches(token1, token2)
    return (token1 == '(' && token2 == ')') || (token1 == '[' && token2 == ']') || (token1 == '{' && token2 == '}') || (token1 == '<' && token2 == '>')
end

function solution1(lines)
    beginnings = "([{<"
    ends = ")]}>"
    total = 0
    for line in lines
        line_list = []
        for char in line
            if in(char, beginnings)
                push!(line_list, char)
            else
                beg = pop!(line_list)
                if !matches(beg, char)
                    total += get_syntax_score(char)
                    break
                end
            end
        end
    end
    return total
end

"""
): 1 point.
]: 2 points.
}: 3 points.
>: 4 points.
"""
function get_completion_token_score(token)
    if token == ')'
        return 1
    elseif token == ']'
        return 2
    elseif token == '}'
        return 3
    elseif token == '>'
        return 4
    end
end

function get_completion_line_score(line_chars)
    total = 0
    for char in line_chars
        total *= 5
        total += get_completion_token_score(char)
    end
    return total
end

complements = Dict()
complements['('] = ')'
complements['['] = ']'
complements['{'] = '}'
complements['<'] = '>'

function solution2(lines)
    beginnings = "([{<"
    ends = ")]}>"
    scores = []

    for line in lines
        broken = false
        line_list = []
        for char in line
            if in(char, beginnings)
                push!(line_list, char)
            else
                beg = pop!(line_list)
                if !matches(beg, char)
                    broken = true
                    break
                end
            end
        end
        if !broken
            completion_chars = []
            for char in reverse(line_list)
                append!(completion_chars, complements[char])
            end
            append!(scores, get_completion_line_score(completion_chars))
        end
    end
    med = round(Int, length(scores) / 2)
    return sort(scores)[med]
end

println(solution1(lines))

println(solution2(lines))