text:
Gm, Gn = 42, 42;
Gcolptr = [1, 3, 7, 11, 14, 16, 20, 25, 29, 32, 36, 41, 45, 51, 55, 59, 64, 68, 72, 76, 81, 84, 87, 92, 96, 100, 103, 105, 109, 114, 118, 122, 127, 131, 135, 140, 144, 146, 148, 151, 154, 157, 159];
Growval = [1, 2, 1, 2, 3, 6, 2, 3, 4, 7, 3, 4, 8, 5, 9, 2, 6, 7, 10, 3, 6, 7, 8, 11, 4, 7, 8, 12, 5, 9, 13, 6, 10, 11, 15, 7, 10, 11, 12, 16, 8, 11, 12, 17, 9, 13, 14, 18, 23, 28, 13, 14, 15, 18, 10, 14, 15, 16, 11, 15, 16, 17, 19, 12, 16, 17, 20, 13, 14, 18, 23, 16, 19, 20, 25, 17, 19, 20, 21, 26, 20, 21, 22, 21, 22, 27, 13, 18, 23, 24, 29, 23, 24, 25, 30, 19, 24, 25, 26, 20, 25, 26, 22, 27, 13, 28, 29, 31, 23, 28, 29, 30, 32, 24, 29, 30, 33, 28, 31, 32, 34, 29, 31, 32, 33, 35, 30, 32, 33, 36, 31, 34, 35, 40, 32, 34, 35, 36, 41, 33, 35, 36, 37, 36, 37, 38, 39, 38, 39, 42, 34, 40, 41, 35, 40, 41, 39, 42];
Gnzval = [-1.0666666666666667, 0.4, 0.4, -1.0857142857142859, 0.2857142857142857, 0.4, 0.2857142857142857, -0.8412698412698412, 0.2222222222222222, 0.3333333333333333, 0.2222222222222222, -0.5079365079365079, 0.2857142857142857, -1.0666666666666667, 0.4, 0.4, -1.5666666666666667, 0.5, 0.6666666666666666, 0.3333333333333333, 0.5, -2.0, 0.5, 0.6666666666666666, 0.2857142857142857, 0.5, -1.4523809523809523, 0.6666666666666666, 0.4, -0.6857142857142857, 0.2857142857142857, 0.6666666666666666, -2.6666666666666665, 1.0, 1.0, 0.6666666666666666, 1.0, -3.6666666666666665, 1.0, 1.0, 0.6666666666666666, 1.0, -2.6666666666666665, 1.0, 0.2857142857142857, -1.4228937728937727, 0.4, 0.3333333333333333, 0.25, 0.15384615384615385, 0.4, -2.4, 1.0, 1.0, 1.0, 1.0, -3.0, 1.0, 1.0, 1.0, -4.0, 1.0, 1.0, 1.0, 1.0, -3.0, 1.0, 0.3333333333333333, 1.0, -2.0, 0.6666666666666666, 1.0, -2.6666666666666665, 1.0, 0.6666666666666666, 1.0, 1.0, -2.9166666666666665, 0.25, 0.6666666666666666, 0.25, -0.7222222222222222, 0.25, 0.25, -0.9166666666666666, 0.6666666666666666, 0.25, 0.6666666666666666, -2.083333333333333, 0.5, 0.6666666666666666, 0.5, -1.6666666666666665, 0.5, 0.6666666666666666, 0.6666666666666666, 0.5, -1.6666666666666665, 0.5, 0.6666666666666666, 0.5, -1.6666666666666665, 0.6666666666666666, -1.1666666666666665, 0.15384615384615385, -0.5371794871794872, 0.25, 0.13333333333333333, 0.6666666666666666, 0.25, -2.9166666666666665, 1.0, 1.0, 0.6666666666666666, 1.0, -2.6666666666666665, 1.0, 0.13333333333333333, -0.47320261437908495, 0.2222222222222222, 0.11764705882352941, 1.0, 0.2222222222222222, -3.2222222222222223, 1.0, 1.0, 1.0, 1.0, -3.0, 1.0, 0.11764705882352941, -0.42291021671826623, 0.2, 0.10526315789473684, 1.0, 0.2, -3.2, 1.0, 1.0, 1.0, 1.0, -4.0, 1.0, 1.0, -2.6666666666666665, -2.4, 1.0, 1.0, -1.3333333333333333, 0.3333333333333333, 0.10526315789473684, -0.28708133971291866, 0.18181818181818182, 1.0, 0.18181818181818182, -2.1818181818181817, 0.3333333333333333, -0.5555555555555556];
G = SparseMatrixCSC(Gm, Gn, Gcolptr, Growval, Gnzval);


text:
struct GError
    domain::UInt32
    code::Cint
    message::Ptr{UInt8}
end

ext: jl


text:
include("../src/hypergraph-affinity-functions.jl")
names = ["Clothing, Shoes & Accessories"
"Electronics and Office"
"Home, Furniture & Appliances"
"Home Improvement & Patio"
"Baby"
"Toys, Games, and Video Games"
"Food, Household & Pets"
"Pharmacy, Health & Beauty"
"Sports, Fitness & Outdoors"
"Auto, Tires & Industrial"
"Other"]
dataset = "walmart-trips"
r = 25
H, labels = read_hypergraph_data(dataset,r)

## Categories 1 and 7 make up over 50% of the dataset.
# At a high level, this suggests that a lot of shopping can be broadly
# categorized into grocery shopping or clothes shopping. We take the subset
# of the data corresponding to these two (almost balanced) categories, to obtain
# a two-class hypergraph
CF = findall(x->in(x,[1,7]),labels)
labels = labels[CF]
classes = round.(Int64,(labels .- 1) ./6)
n = length(CF)

H2 = H[:,CF]
order = vec(sum(H2,dims = 2))
nonempty = findall(x->x>0,order)
H = H2[nonempty,:]
# class 1 is food

## Bootstraps
alpha, B1, B2, H1, H2, tR1, tR2, N = Hypergraph_to_Scores(H,classes,25)

for k = 4:4:12
B = 100
TR1 = tR1[k,1:k]  # True affinity scores
TR0 = tR2[k,1:k]

TA1 = H1[k,1:k]  # True affinity scores
TA0 = H2[k,1:k]

# Bootstraps
R0, R1, MR1, MR0, SR1, SR0, A0, A1, MA1, MA0, SA1, SA0 = Bootstrap_Affinities(H,k,B,classes)
## Plot
sc = 2
lw = 2.5
ms = 9
gfs = 15
tfs = 15
titlesize = 14
s1 = 400
s2 = 300
color1 = :purple
color2 = RGB(218/255,165/255,35/255)
title = ""
leg = false
xlab = "Affinity type t"
ylab = "Affinity / Baseline"
p = Plots.plot([1; k],[1; 1],fontfamily = "helvetica",linewidth = 2,yscale = :log10,legend = false,grid = false, size = (s1,s2),color = :gray)
title!(p,title,titlefontsize = titlesize)
xaxis!(p,xlab, xticks = k/4:k/4:k,tickfontsize=tfs,guidefontsize = gfs)
yaxis!(p,ylab,tickfontsize=tfs,guidefontsize = gfs)

plot!(p,1:k,[TR0 TR0], fillrange=[MR0-sc*SR0 MR0+sc*SR0], fillalpha=0.3,color = color2, linewidth = lw, markersize = ms,
    markershape = :circle, markerstrokecolor = color2, label = class2)
plot!(p,1:k,[TR1 TR1], fillrange=[MR1-sc*SR1 MR1+sc*SR1],fillalpha=0.3,color = color1, linewidth = lw, markersize = ms,
    markershape = :circle,markerstrokecolor = color1, label = class1)

savefig("Figures/Ratios_$(k)_Walmart_Food_Clothing.pdf")

## Next
s1 = 400
s2 = 300
title = "$k-size hyperedge"
title = ""
leg = false
xlab = "Affinity type t"
ylab = "Affinity"
p = Plots.plot(yscale = :log10,fontfamily = "helvetica",linewidth = 2,legend = false,grid = false, size = (s1,s2),color = :gray)
title!(p,title,titlefontsize = titlesize)
xaxis!(p,xlab, xticks = k/4:k/4:k,tickfontsize=tfs,guidefontsize = gfs)
yaxis!(p,ylab,tickfontsize=tfs,guidefontsize = gfs)
plot!(p,1:k,[TA1 TA1], fillrange=[MA1-sc*SA1 MA1+sc*SA1],fillalpha=0.3,color = color1, linewidth = lw, markersize = ms,
    markershape = :circle,markerstrokecolor = color1, label = class1)
plot!(p,1:k,[TA0 TA0], fillrange=[MA0-sc*SA0 MA0+sc*SA0], fillalpha=0.3,color = color2, linewidth = lw, markersize = ms,
    markershape = :circle, markerstrokecolor = color2, label = class2)

savefig("Figures/Affinities_$(k)_Walmart_Food_Clothing_log.pdf")
end
## Group Homophily Index, print usable LaTeX code for building table
# include("../src/hypergraph-affinity-functions.jl")
# S1,S2  = GHI(H,classes,14)

max_stars_repo_name: nveldt/HypergraphHomophily

max_issues_repo_name: nveldt/HypergraphHomophily

max_forks_repo_name: nveldt/HypergraphHomophily


text:
using JUDI, SegyIO, LinearAlgebra, PyPlot

# Set up model structure
n = (120, 100)   # (x,y,z) or (x,z)
d = (10., 10.)
o = (0., 0.)

# Velocity [km/s]
v = ones(Float32,n) .+ 0.4f0
v[:,Int(round(end/2)):end] .= 3f0
v0 = ones(Float32,n) .+ 0.4f0

# Slowness squared [s^2/km^2]
m0 = (1f0 ./ v0).^2
m = (1f0 ./ v).^2
dm = vec(m - m0)

# Setup info and model structure
nsrc = 2	# number of sources
model0 = Model(n, d, o, m0)
model = Model(n, d, o, m)

# Set up receiver geometry
nxrec = 120
xrec = range(50f0, stop=1150f0, length=nxrec)
yrec = 0f0
zrec = range(50f0, stop=50f0, length=nxrec)

# receiver sampling and recording time
time = 1000f0   # receiver recording time [ms]
dt = 4f0    # receiver sampling interval [ms]

# Set up receiver structure
recGeometry = Geometry(xrec, yrec, zrec; dt=dt, t=time, nsrc=nsrc)

# Source wavelet
f0 = 0.01f0     # kHz
wavelet = ricker_wavelet(time, dt, f0)

# Set up info structure for linear operators
ntComp = get_computational_nt(recGeometry, model)
info = Info(prod(n), nsrc, ntComp)

###################################################################################################

# Write shots as segy files to disk
opt = Options(return_array=false, dt_comp=1.0, free_surface=true)

# Setup operators
Pr = judiProjection(info, recGeometry)
F = judiModeling(info, model; options=opt)

# Extended source weights
weights = Array{Array}(undef, nsrc)
for j=1:nsrc
    weights[j] = randn(Float32, model.n)
end
w = judiWeights(weights)

# Create operator for injecting the weights, multiplied by the provided wavelet(s)
Pw = judiLRWF(info, wavelet)

# Model observed data w/ extended source
F = Pr*F*adjoint(Pw)

# Simultaneous observed data
d_sim = F*w
dw = adjoint(F)*d_sim

# Jacobian
J = judiJacobian(F, w)
d_lin = J*dm
g = adjoint(J)*d_lin

# Plot results
figure()
subplot(1,2,1)
imshow(d_sim.data[1], vmin=-5e2, vmax=5e2, cmap="gray"); title("Non-linear shot record")
subplot(1,2,2)
imshow(d_lin.data[1], vmin=-5e3, vmax=5e3, cmap="gray"); title("Linearized shot record")

figure()
subplot(1,2,1)
imshow(adjoint(dw.weights[1]), vmin=-5e6, vmax=5e6, cmap="gray"); title("Weights 1")
subplot(1,2,2)
imshow(adjoint(reshape(g, model0.n)), vmin=-1e8, vmax=1e8, cmap="gray"); title("Gradient w.r.t. m")

max_stars_repo_name: nogueirapeterson/JUDI

max_issues_repo_name: nogueirapeterson/JUDI

max_forks_repo_name: nogueirapeterson/JUDI


text:
"""
	$(SIGNATURES)

Show a text file on the screen.
"""
function show_text_file(filePath; io = stdout)
    _, fName = splitdir(filePath);
    println(io, "----------  $fName  ----------");
    for line in eachline(filePath)
        println(io, line);
    end
    println(io, "----------  [end] $fName  ----------");
    return nothing;
end

"""
    $(SIGNATURES)

Display string vector on fixed with screen.
"""
function show_string_vector(sV :: Vector{T1},  width :: Integer = 80) where
    T1 <: AbstractString

    n = length(sV);
    if n < 1
        return nothing
    end

    iCol = 0;
    for s in sV
        len1 = length(s);
        if len1 > 0
            if iCol + len1 > width
                println(" ")
                iCol = 0;
            end
            print(s);
            iCol = iCol + len1;
            print("    ");
            iCol = iCol + 4;
        end
    end
    println(" ")
end

function string_vector_to_lines(sV :: Vector{T1}, width :: Integer = 80) where
    T1 <: AbstractString

    lineV = Vector{T1}();
    if isempty(sV)
        return lineV
    end

    n = length(sV);

    line = "";
    for s in sV
        if !isempty(s)
            if length(s) + length(line) > width
                # Start new line
                push!(lineV, line);
                line = "";
            end
            line = line * s * "    ";
        end
    end
    push!(lineV, line);
    return lineV
end

## ------------  Print to multiple IO streams

"""
	$(SIGNATURES)

Object that holds multiple IO streams, so that a single `print` statement can be directed to all of them. Main use case: selectively write to optimization log and stdout.
This must be a subtype of IO for dispatch to work correctly.

Based on https://discourse.julialang.org/t/issues-with-println-buffering-output-when-redirecting-stdout-to-a-txt-file/23738/13
"""
struct MultiIO{T} <: IO
    ioV :: Vector{T}
end

function Base.print(io :: MultiIO, args...)
    for io in io.ioV
        print(io, args...);
    end
end

# Avoid method ambiguity
function Base.print(io :: MultiIO, s :: Union{SubString{String}, String})
    for io in io.ioV
        print(io, s);
    end
end

function Base.println(io :: MultiIO, args...)
    for io in io.ioV
        println(io, args...);
    end
end

# Avoid method ambiguity
function Base.println(io :: MultiIO, s :: Union{SubString{String}, String})
    for io in io.ioV
        println(io, s);
    end
end

function print_flush(io :: MultiIO, args...)
    print(io, args...);
    flush(io);
end

function println_flush(io :: MultiIO, args...)
    println(io, args...);
    flush(io);
end

function Base.flush(io :: MultiIO)
    for io in io.ioV
        flush(io);
    end
end

# """
# Print numeric vector (or array with 1 dim)

# IN
#     fmtStr
#         formatting string that works with Formatting package
# """
# function sprintf(fmtStr :: String, x :: Array{T}) where T <: Number
#     x2 = vec(x)
#     outStr = "";
#     for xVal in x2
#       outStr = outStr * Formatting.format(fmtStr, xVal)
#     end
#     return outStr
# end

# # Scalar input
# function sprintf(fmtStr :: String, x :: T1) where T1 <: Number
#     return sprintf(fmtStr, [x])
# end

# function printf(fmtStr :: String,  x :: Array{T}) where T <: Number
#     print(sprintf(fmtStr, x));
#     return nothing
# end

# ---------------------


text:
#==============================================================================#
# CloudTrail.jl
#
# This file is generated from:
# https://github.com/aws/aws-sdk-js/blob/master/apis/cloudtrail-2013-11-01.normal.json
#==============================================================================#

__precompile__()

module CloudTrail

using AWSCore

"""
    using AWSSDK.CloudTrail.add_tags
    add_tags([::AWSConfig], arguments::Dict)
    add_tags([::AWSConfig]; ResourceId=, <keyword arguments>)

    using AWSCore.Services.cloudtrail
    cloudtrail([::AWSConfig], "AddTags", arguments::Dict)
    cloudtrail([::AWSConfig], "AddTags", ResourceId=, <keyword arguments>)

# AddTags Operation

Adds one or more tags to a trail, up to a limit of 50. Tags must be unique per trail. Overwrites an existing tag's value when a new value is specified for an existing tag key. If you specify a key without a value, the tag will be created with the specified key and a value of null. You can tag a trail that applies to all regions only from the region in which the trail was created (that is, from its home region).

# Arguments

## `ResourceId = ::String` -- *Required*
Specifies the ARN of the trail to which one or more tags will be added. The format of a trail ARN is:

`arn:aws:cloudtrail:us-east-1:123456789012:trail/MyTrail`

## `TagsList = [[ ... ], ...]`
Contains a list of CloudTrail tags, up to a limit of 50
```
 TagsList = [[
        "Key" => <required> ::String,
        "Value" =>  ::String
    ], ...]
```

# Returns

`AddTagsResponse`

# Exceptions

`ResourceNotFoundException`, `CloudTrailARNInvalidException`, `ResourceTypeNotSupportedException`, `TagsLimitExceededException`, `InvalidTrailNameException`, `InvalidTagParameterException`, `UnsupportedOperationException` or `OperationNotPermittedException`.

See also: [AWS API Documentation](https://docs.aws.amazon.com/goto/WebAPI/cloudtrail-2013-11-01/AddTags)
"""
@inline add_tags(aws::AWSConfig=default_aws_config(); args...) = add_tags(aws, args)

@inline add_tags(aws::AWSConfig, args) = AWSCore.Services.cloudtrail(aws, "AddTags", args)

@inline add_tags(args) = add_tags(default_aws_config(), args)

"""
    using AWSSDK.CloudTrail.create_trail
    create_trail([::AWSConfig], arguments::Dict)
    create_trail([::AWSConfig]; Name=, S3BucketName=, <keyword arguments>)

    using AWSCore.Services.cloudtrail
    cloudtrail([::AWSConfig], "CreateTrail", arguments::Dict)
    cloudtrail([::AWSConfig], "CreateTrail", Name=, S3BucketName=, <keyword arguments>)

# CreateTrail Operation

Creates a trail that specifies the settings for delivery of log data to an Amazon S3 bucket. A maximum of five trails can exist in a region, irrespective of the region in which they were created.

# Arguments

## `Name = ::String` -- *Required*
Specifies the name of the trail. The name must meet the following requirements:

*   Contain only ASCII letters (a-z, A-Z), numbers (0-9), periods (.), underscores (_), or dashes (-)

*   Start with a letter or number, and end with a letter or number

*   Be between 3 and 128 characters

*   Have no adjacent periods, underscores or dashes. Names like `my-_namespace` and `my--namespace` are invalid.

*   Not be in IP address format (for example, 192.168.5.4)

## `S3BucketName = ::String` -- *Required*
Specifies the name of the Amazon S3 bucket designated for publishing log files. See [Amazon S3 Bucket Naming Requirements](http://docs.aws.amazon.com/awscloudtrail/latest/userguide/create_trail_naming_policy.html).

## `S3KeyPrefix = ::String`
Specifies the Amazon S3 key prefix that comes after the name of the bucket you have designated for log file delivery. For more information, see [Finding Your CloudTrail Log Files](http://docs.aws.amazon.com/awscloudtrail/latest/userguide/cloudtrail-find-log-files.html). The maximum length is 200 characters.

## `SnsTopicName = ::String`
Specifies the name of the Amazon SNS topic defined for notification of log file delivery. The maximum length is 256 characters.

## `IncludeGlobalServiceEvents = ::Bool`
Specifies whether the trail is publishing events from global services such as IAM to the log files.

## `IsMultiRegionTrail = ::Bool`
Specifies whether the trail is created in the current region or in all regions. The default is false.

## `EnableLogFileValidation = ::Bool`
Specifies whether log file integrity validation is enabled. The default is false.

**Note**
> When you disable log file integrity validation, the chain of digest files is broken after one hour. CloudTrail will not create digest files for log files that were delivered during a period in which log file integrity validation was disabled. For example, if you enable log file integrity validation at noon on January 1, disable it at noon on January 2, and re-enable it at noon on January 10, digest files will not be created for the log files delivered from noon on January 2 to noon on January 10. The same applies whenever you stop CloudTrail logging or delete a trail.

## `CloudWatchLogsLogGroupArn = ::String`
Specifies a log group name using an Amazon Resource Name (ARN), a unique identifier that represents the log group to which CloudTrail logs will be delivered. Not required unless you specify CloudWatchLogsRoleArn.

## `CloudWatchLogsRoleArn = ::String`
Specifies the role for the CloudWatch Logs endpoint to assume to write to a user's log group.

## `KmsKeyId = ::String`
Specifies the KMS key ID to use to encrypt the logs delivered by CloudTrail. The value can be an alias name prefixed by "alias/", a fully specified ARN to an alias, a fully specified ARN to a key, or a globally unique identifier.

Examples:

*   alias/MyAliasName

*   arn:aws:kms:us-east-1:123456789012:alias/MyAliasName

*   arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012

*   12345678-1234-1234-1234-123456789012

# Returns

`CreateTrailResponse`

# Exceptions

`MaximumNumberOfTrailsExceededException`, `TrailAlreadyExistsException`, `S3BucketDoesNotExistException`, `InsufficientS3BucketPolicyException`, `InsufficientSnsTopicPolicyException`, `InsufficientEncryptionPolicyException`, `InvalidS3BucketNameException`, `InvalidS3PrefixException`, `InvalidSnsTopicNameException`, `InvalidKmsKeyIdException`, `InvalidTrailNameException`, `TrailNotProvidedException`, `InvalidParameterCombinationException`, `KmsKeyNotFoundException`, `KmsKeyDisabledException`, `KmsException`, `InvalidCloudWatchLogsLogGroupArnException`, `InvalidCloudWatchLogsRoleArnException`, `CloudWatchLogsDeliveryUnavailableException`, `UnsupportedOperationException` or `OperationNotPermittedException`.

See also: [AWS API Documentation](https://docs.aws.amazon.com/goto/WebAPI/cloudtrail-2013-11-01/CreateTrail)
"""
@inline create_trail(aws::AWSConfig=default_aws_config(); args...) = create_trail(aws, args)

@inline create_trail(aws::AWSConfig, args) = AWSCore.Services.cloudtrail(aws, "CreateTrail", args)

@inline create_trail(args) = create_trail(default_aws_config(), args)

"""
    using AWSSDK.CloudTrail.delete_trail
    delete_trail([::AWSConfig], arguments::Dict)
    delete_trail([::AWSConfig]; Name=)

    using AWSCore.Services.cloudtrail
    cloudtrail([::AWSConfig], "DeleteTrail", arguments::Dict)
    cloudtrail([::AWSConfig], "DeleteTrail", Name=)

# DeleteTrail Operation

Deletes a trail. This operation must be called from the region in which the trail was created. `DeleteTrail` cannot be called on the shadow trails (replicated trails in other regions) of a trail that is enabled in all regions.

# Arguments

## `Name = ::String` -- *Required*
Specifies the name or the CloudTrail ARN of the trail to be deleted. The format of a trail ARN is: `arn:aws:cloudtrail:us-east-1:123456789012:trail/MyTrail`

# Returns

`DeleteTrailResponse`

# Exceptions

`TrailNotFoundException`, `InvalidTrailNameException` or `InvalidHomeRegionException`.

See also: [AWS API Documentation](https://docs.aws.amazon.com/goto/WebAPI/cloudtrail-2013-11-01/DeleteTrail)
"""
@inline delete_trail(aws::AWSConfig=default_aws_config(); args...) = delete_trail(aws, args)

@inline delete_trail(aws::AWSConfig, args) = AWSCore.Services.cloudtrail(aws, "DeleteTrail", args)

@inline delete_trail(args) = delete_trail(default_aws_config(), args)

"""
    using AWSSDK.CloudTrail.describe_trails
    describe_trails([::AWSConfig], arguments::Dict)
    describe_trails([::AWSConfig]; <keyword arguments>)

    using AWSCore.Services.cloudtrail
    cloudtrail([::AWSConfig], "DescribeTrails", arguments::Dict)
    cloudtrail([::AWSConfig], "DescribeTrails", <keyword arguments>)

# DescribeTrails Operation

Retrieves settings for the trail associated with the current region for your account.

# Arguments

## `trailNameList = [::String, ...]`
Specifies a list of trail names, trail ARNs, or both, of the trails to describe. The format of a trail ARN is:

`arn:aws:cloudtrail:us-east-1:123456789012:trail/MyTrail`

If an empty list is specified, information for the trail in the current region is returned.

*   If an empty list is specified and `IncludeShadowTrails` is false, then information for all trails in the current region is returned.

*   If an empty list is specified and IncludeShadowTrails is null or true, then information for all trails in the current region and any associated shadow trails in other regions is returned.

**Note**
> If one or more trail names are specified, information is returned only if the names match the names of trails belonging only to the current region. To return information about a trail in another region, you must specify its trail ARN.

## `includeShadowTrails = ::Bool`
Specifies whether to include shadow trails in the response. A shadow trail is the replication in a region of a trail that was created in a different region. The default is true.

# Returns

`DescribeTrailsResponse`

# Exceptions

`UnsupportedOperationException` or `OperationNotPermittedException`.

See also: [AWS API Documentation](https://docs.aws.amazon.com/goto/WebAPI/cloudtrail-2013-11-01/DescribeTrails)
"""
@inline describe_trails(aws::AWSConfig=default_aws_config(); args...) = describe_trails(aws, args)

@inline describe_trails(aws::AWSConfig, args) = AWSCore.Services.cloudtrail(aws, "DescribeTrails", args)

@inline describe_trails(args) = describe_trails(default_aws_config(), args)

"""
    using AWSSDK.CloudTrail.get_event_selectors
    get_event_selectors([::AWSConfig], arguments::Dict)
    get_event_selectors([::AWSConfig]; TrailName=)

    using AWSCore.Services.cloudtrail
    cloudtrail([::AWSConfig], "GetEventSelectors", arguments::Dict)
    cloudtrail([::AWSConfig], "GetEventSelectors", TrailName=)

# GetEventSelectors Operation

Describes the settings for the event selectors that you configured for your trail. The information returned for your event selectors includes the following:

*   The S3 objects that you are logging for data events.

*   If your event selector includes management events.

*   If your event selector includes read-only events, write-only events, or all.

For more information, see [Logging Data and Management Events for Trails](http://docs.aws.amazon.com/awscloudtrail/latest/userguide/logging-management-and-data-events-with-cloudtrail.html) in the *AWS CloudTrail User Guide*.

# Arguments

## `TrailName = ::String` -- *Required*
Specifies the name of the trail or trail ARN. If you specify a trail name, the string must meet the following requirements:

*   Contain only ASCII letters (a-z, A-Z), numbers (0-9), periods (.), underscores (_), or dashes (-)

*   Start with a letter or number, and end with a letter or number

*   Be between 3 and 128 characters

*   Have no adjacent periods, underscores or dashes. Names like `my-_namespace` and `my--namespace` are invalid.

*   Not be in IP address format (for example, 192.168.5.4)

If you specify a trail ARN, it must be in the format:

`arn:aws:cloudtrail:us-east-1:123456789012:trail/MyTrail`

# Returns

`GetEventSelectorsResponse`

# Exceptions

`TrailNotFoundException`, `InvalidTrailNameException`, `UnsupportedOperationException` or `OperationNotPermittedException`.

See also: [AWS API Documentation](https://docs.aws.amazon.com/goto/WebAPI/cloudtrail-2013-11-01/GetEventSelectors)
"""
@inline get_event_selectors(aws::AWSConfig=default_aws_config(); args...) = get_event_selectors(aws, args)

@inline get_event_selectors(aws::AWSConfig, args) = AWSCore.Services.cloudtrail(aws, "GetEventSelectors", args)

@inline get_event_selectors(args) = get_event_selectors(default_aws_config(), args)

"""
    using AWSSDK.CloudTrail.get_trail_status
    get_trail_status([::AWSConfig], arguments::Dict)
    get_trail_status([::AWSConfig]; Name=)

    using AWSCore.Services.cloudtrail
    cloudtrail([::AWSConfig], "GetTrailStatus", arguments::Dict)
    cloudtrail([::AWSConfig], "GetTrailStatus", Name=)

# GetTrailStatus Operation

Returns a JSON-formatted list of information about the specified trail. Fields include information on delivery errors, Amazon SNS and Amazon S3 errors, and start and stop logging times for each trail. This operation returns trail status from a single region. To return trail status from all regions, you must call the operation on each region.

# Arguments

## `Name = ::String` -- *Required*
Specifies the name or the CloudTrail ARN of the trail for which you are requesting status. To get the status of a shadow trail (a replication of the trail in another region), you must specify its ARN. The format of a trail ARN is:

`arn:aws:cloudtrail:us-east-1:123456789012:trail/MyTrail`

# Returns

`GetTrailStatusResponse`

# Exceptions

`TrailNotFoundException` or `InvalidTrailNameException`.

See also: [AWS API Documentation](https://docs.aws.amazon.com/goto/WebAPI/cloudtrail-2013-11-01/GetTrailStatus)
"""
@inline get_trail_status(aws::AWSConfig=default_aws_config(); args...) = get_trail_status(aws, args)

@inline get_trail_status(aws::AWSConfig, args) = AWSCore.Services.cloudtrail(aws, "GetTrailStatus", args)

@inline get_trail_status(args) = get_trail_status(default_aws_config(), args)

"""
    using AWSSDK.CloudTrail.list_public_keys
    list_public_keys([::AWSConfig], arguments::Dict)
    list_public_keys([::AWSConfig]; <keyword arguments>)

    using AWSCore.Services.cloudtrail
    cloudtrail([::AWSConfig], "ListPublicKeys", arguments::Dict)
    cloudtrail([::AWSConfig], "ListPublicKeys", <keyword arguments>)

# ListPublicKeys Operation

Returns all public keys whose private keys were used to sign the digest files within the specified time range. The public key is needed to validate digest files that were signed with its corresponding private key.

**Note**
> CloudTrail uses different private/public key pairs per region. Each digest file is signed with a private key unique to its region. Therefore, when you validate a digest file from a particular region, you must look in the same region for its corresponding public key.

# Arguments

## `StartTime = timestamp`
Optionally specifies, in UTC, the start of the time range to look up public keys for CloudTrail digest files. If not specified, the current time is used, and the current public key is returned.

## `EndTime = timestamp`
Optionally specifies, in UTC, the end of the time range to look up public keys for CloudTrail digest files. If not specified, the current time is used.

## `NextToken = ::String`
Reserved for future use.

# Returns

`ListPublicKeysResponse`

# Exceptions

`InvalidTimeRangeException`, `UnsupportedOperationException`, `OperationNotPermittedException` or `InvalidTokenException`.

See also: [AWS API Documentation](https://docs.aws.amazon.com/goto/WebAPI/cloudtrail-2013-11-01/ListPublicKeys)
"""
@inline list_public_keys(aws::AWSConfig=default_aws_config(); args...) = list_public_keys(aws, args)

@inline list_public_keys(aws::AWSConfig, args) = AWSCore.Services.cloudtrail(aws, "ListPublicKeys", args)

@inline list_public_keys(args) = list_public_keys(default_aws_config(), args)

"""
    using AWSSDK.CloudTrail.list_tags
    list_tags([::AWSConfig], arguments::Dict)
    list_tags([::AWSConfig]; ResourceIdList=, <keyword arguments>)

    using AWSCore.Services.cloudtrail
    cloudtrail([::AWSConfig], "ListTags", arguments::Dict)
    cloudtrail([::AWSConfig], "ListTags", ResourceIdList=, <keyword arguments>)

# ListTags Operation

Lists the tags for the trail in the current region.

# Arguments

## `ResourceIdList = [::String, ...]` -- *Required*
Specifies a list of trail ARNs whose tags will be listed. The list has a limit of 20 ARNs. The format of a trail ARN is:

`arn:aws:cloudtrail:us-east-1:123456789012:trail/MyTrail`

## `NextToken = ::String`
Reserved for future use.

# Returns

`ListTagsResponse`

# Exceptions

`ResourceNotFoundException`, `CloudTrailARNInvalidException`, `ResourceTypeNotSupportedException`, `InvalidTrailNameException`, `UnsupportedOperationException`, `OperationNotPermittedException` or `InvalidTokenException`.

See also: [AWS API Documentation](https://docs.aws.amazon.com/goto/WebAPI/cloudtrail-2013-11-01/ListTags)
"""
@inline list_tags(aws::AWSConfig=default_aws_config(); args...) = list_tags(aws, args)

@inline list_tags(aws::AWSConfig, args) = AWSCore.Services.cloudtrail(aws, "ListTags", args)

@inline list_tags(args) = list_tags(default_aws_config(), args)

"""
    using AWSSDK.CloudTrail.lookup_events
    lookup_events([::AWSConfig], arguments::Dict)
    lookup_events([::AWSConfig]; <keyword arguments>)

    using AWSCore.Services.cloudtrail
    cloudtrail([::AWSConfig], "LookupEvents", arguments::Dict)
    cloudtrail([::AWSConfig], "LookupEvents", <keyword arguments>)

# LookupEvents Operation

Looks up API activity events captured by CloudTrail that create, update, or delete resources in your account. Events for a region can be looked up for the times in which you had CloudTrail turned on in that region during the last seven days. Lookup supports the following attributes:

*   Event ID

*   Event name

*   Event source

*   Resource name

*   Resource type

*   User name

All attributes are optional. The default number of results returned is 10, with a maximum of 50 possible. The response includes a token that you can use to get the next page of results.

**Important**
> The rate of lookup requests is limited to one per second per account. If this limit is exceeded, a throttling error occurs.

**Important**
> Events that occurred during the selected time range will not be available for lookup if CloudTrail logging was not enabled when the events occurred.

# Arguments

## `LookupAttributes = [[ ... ], ...]`
Contains a list of lookup attributes. Currently the list can contain only one item.
```
 LookupAttributes = [[
        "AttributeKey" => <required> "EventId", "EventName", "Username", "ResourceType", "ResourceName" or "EventSource",
        "AttributeValue" => <required> ::String
    ], ...]
```

## `StartTime = timestamp`
Specifies that only events that occur after or at the specified time are returned. If the specified start time is after the specified end time, an error is returned.

## `EndTime = timestamp`
Specifies that only events that occur before or at the specified time are returned. If the specified end time is before the specified start time, an error is returned.

## `MaxResults = ::Int`
The number of events to return. Possible values are 1 through 50. The default is 10.

## `NextToken = ::String`
The token to use to get the next page of results after a previous API call. This token must be passed in with the same parameters that were specified in the the original call. For example, if the original call specified an AttributeKey of 'Username' with a value of 'root', the call with NextToken should include those same parameters.

# Returns

`LookupEventsResponse`

# Exceptions

`InvalidLookupAttributesException`, `InvalidTimeRangeException`, `InvalidMaxResultsException` or `InvalidNextTokenException`.

See also: [AWS API Documentation](https://docs.aws.amazon.com/goto/WebAPI/cloudtrail-2013-11-01/LookupEvents)
"""
@inline lookup_events(aws::AWSConfig=default_aws_config(); args...) = lookup_events(aws, args)

@inline lookup_events(aws::AWSConfig, args) = AWSCore.Services.cloudtrail(aws, "LookupEvents", args)

@inline lookup_events(args) = lookup_events(default_aws_config(), args)

"""
    using AWSSDK.CloudTrail.put_event_selectors
    put_event_selectors([::AWSConfig], arguments::Dict)
    put_event_selectors([::AWSConfig]; TrailName=, EventSelectors=)

    using AWSCore.Services.cloudtrail
    cloudtrail([::AWSConfig], "PutEventSelectors", arguments::Dict)
    cloudtrail([::AWSConfig], "PutEventSelectors", TrailName=, EventSelectors=)

# PutEventSelectors Operation

Configures an event selector for your trail. Use event selectors to specify whether you want your trail to log management and/or data events. When an event occurs in your account, CloudTrail evaluates the event selectors in all trails. For each trail, if the event matches any event selector, the trail processes and logs the event. If the event doesn't match any event selector, the trail doesn't log the event.

Example

1.  You create an event selector for a trail and specify that you want write-only events.

2.  The EC2 `GetConsoleOutput` and `RunInstances` API operations occur in your account.

3.  CloudTrail evaluates whether the events match your event selectors.

4.  The `RunInstances` is a write-only event and it matches your event selector. The trail logs the event.

5.  The `GetConsoleOutput` is a read-only event but it doesn't match your event selector. The trail doesn't log the event.

The `PutEventSelectors` operation must be called from the region in which the trail was created; otherwise, an `InvalidHomeRegionException` is thrown.

You can configure up to five event selectors for each trail. For more information, see [Logging Data and Management Events for Trails](http://docs.aws.amazon.com/awscloudtrail/latest/userguide/logging-management-and-data-events-with-cloudtrail.html) in the *AWS CloudTrail User Guide*.

# Arguments

## `TrailName = ::String` -- *Required*
Specifies the name of the trail or trail ARN. If you specify a trail name, the string must meet the following requirements:

*   Contain only ASCII letters (a-z, A-Z), numbers (0-9), periods (.), underscores (_), or dashes (-)

*   Start with a letter or number, and end with a letter or number

*   Be between 3 and 128 characters

*   Have no adjacent periods, underscores or dashes. Names like `my-_namespace` and `my--namespace` are invalid.

*   Not be in IP address format (for example, 192.168.5.4)

If you specify a trail ARN, it must be in the format:

`arn:aws:cloudtrail:us-east-1:123456789012:trail/MyTrail`

## `EventSelectors = [[ ... ], ...]` -- *Required*
Specifies the settings for your event selectors. You can configure up to five event selectors for a trail.
```
 EventSelectors = [[
        "ReadWriteType" =>  "ReadOnly", "WriteOnly" or "All",
        "IncludeManagementEvents" =>  ::Bool,
        "DataResources" =>  [[
            "Type" =>  ::String,
            "Values" =>  [::String, ...]
        ], ...]
    ], ...]
```

# Returns

`PutEventSelectorsResponse`

# Exceptions

`TrailNotFoundException`, `InvalidTrailNameException`, `InvalidHomeRegionException`, `InvalidEventSelectorsException`, `UnsupportedOperationException` or `OperationNotPermittedException`.

See also: [AWS API Documentation](https://docs.aws.amazon.com/goto/WebAPI/cloudtrail-2013-11-01/PutEventSelectors)
"""
@inline put_event_selectors(aws::AWSConfig=default_aws_config(); args...) = put_event_selectors(aws, args)

@inline put_event_selectors(aws::AWSConfig, args) = AWSCore.Services.cloudtrail(aws, "PutEventSelectors", args)

@inline put_event_selectors(args) = put_event_selectors(default_aws_config(), args)

"""
    using AWSSDK.CloudTrail.remove_tags
    remove_tags([::AWSConfig], arguments::Dict)
    remove_tags([::AWSConfig]; ResourceId=, <keyword arguments>)

    using AWSCore.Services.cloudtrail
    cloudtrail([::AWSConfig], "RemoveTags", arguments::Dict)
    cloudtrail([::AWSConfig], "RemoveTags", ResourceId=, <keyword arguments>)

# RemoveTags Operation

Removes the specified tags from a trail.

# Arguments

## `ResourceId = ::String` -- *Required*
Specifies the ARN of the trail from which tags should be removed. The format of a trail ARN is:

`arn:aws:cloudtrail:us-east-1:123456789012:trail/MyTrail`

## `TagsList = [[ ... ], ...]`
Specifies a list of tags to be removed.
```
 TagsList = [[
        "Key" => <required> ::String,
        "Value" =>  ::String
    ], ...]
```

# Returns

`RemoveTagsResponse`

# Exceptions

`ResourceNotFoundException`, `CloudTrailARNInvalidException`, `ResourceTypeNotSupportedException`, `InvalidTrailNameException`, `InvalidTagParameterException`, `UnsupportedOperationException` or `OperationNotPermittedException`.

See also: [AWS API Documentation](https://docs.aws.amazon.com/goto/WebAPI/cloudtrail-2013-11-01/RemoveTags)
"""
@inline remove_tags(aws::AWSConfig=default_aws_config(); args...) = remove_tags(aws, args)

@inline remove_tags(aws::AWSConfig, args) = AWSCore.Services.cloudtrail(aws, "RemoveTags", args)

@inline remove_tags(args) = remove_tags(default_aws_config(), args)

"""
    using AWSSDK.CloudTrail.start_logging
    start_logging([::AWSConfig], arguments::Dict)
    start_logging([::AWSConfig]; Name=)

    using AWSCore.Services.cloudtrail
    cloudtrail([::AWSConfig], "StartLogging", arguments::Dict)
    cloudtrail([::AWSConfig], "StartLogging", Name=)

# StartLogging Operation

Starts the recording of AWS API calls and log file delivery for a trail. For a trail that is enabled in all regions, this operation must be called from the region in which the trail was created. This operation cannot be called on the shadow trails (replicated trails in other regions) of a trail that is enabled in all regions.

# Arguments

## `Name = ::String` -- *Required*
Specifies the name or the CloudTrail ARN of the trail for which CloudTrail logs AWS API calls. The format of a trail ARN is:

`arn:aws:cloudtrail:us-east-1:123456789012:trail/MyTrail`

# Returns

`StartLoggingResponse`

# Exceptions

`TrailNotFoundException`, `InvalidTrailNameException` or `InvalidHomeRegionException`.

See also: [AWS API Documentation](https://docs.aws.amazon.com/goto/WebAPI/cloudtrail-2013-11-01/StartLogging)
"""
@inline start_logging(aws::AWSConfig=default_aws_config(); args...) = start_logging(aws, args)

@inline start_logging(aws::AWSConfig, args) = AWSCore.Services.cloudtrail(aws, "StartLogging", args)

@inline start_logging(args) = start_logging(default_aws_config(), args)

"""
    using AWSSDK.CloudTrail.stop_logging
    stop_logging([::AWSConfig], arguments::Dict)
    stop_logging([::AWSConfig]; Name=)

    using AWSCore.Services.cloudtrail
    cloudtrail([::AWSConfig], "StopLogging", arguments::Dict)
    cloudtrail([::AWSConfig], "StopLogging", Name=)

# StopLogging Operation

Suspends the recording of AWS API calls and log file delivery for the specified trail. Under most circumstances, there is no need to use this action. You can update a trail without stopping it first. This action is the only way to stop recording. For a trail enabled in all regions, this operation must be called from the region in which the trail was created, or an `InvalidHomeRegionException` will occur. This operation cannot be called on the shadow trails (replicated trails in other regions) of a trail enabled in all regions.

# Arguments

## `Name = ::String` -- *Required*
Specifies the name or the CloudTrail ARN of the trail for which CloudTrail will stop logging AWS API calls. The format of a trail ARN is:

`arn:aws:cloudtrail:us-east-1:123456789012:trail/MyTrail`

# Returns

`StopLoggingResponse`

# Exceptions

`TrailNotFoundException`, `InvalidTrailNameException` or `InvalidHomeRegionException`.

See also: [AWS API Documentation](https://docs.aws.amazon.com/goto/WebAPI/cloudtrail-2013-11-01/StopLogging)
"""
@inline stop_logging(aws::AWSConfig=default_aws_config(); args...) = stop_logging(aws, args)

@inline stop_logging(aws::AWSConfig, args) = AWSCore.Services.cloudtrail(aws, "StopLogging", args)

@inline stop_logging(args) = stop_logging(default_aws_config(), args)

"""
    using AWSSDK.CloudTrail.update_trail
    update_trail([::AWSConfig], arguments::Dict)
    update_trail([::AWSConfig]; Name=, <keyword arguments>)

    using AWSCore.Services.cloudtrail
    cloudtrail([::AWSConfig], "UpdateTrail", arguments::Dict)
    cloudtrail([::AWSConfig], "UpdateTrail", Name=, <keyword arguments>)

# UpdateTrail Operation

Updates the settings that specify delivery of log files. Changes to a trail do not require stopping the CloudTrail service. Use this action to designate an existing bucket for log delivery. If the existing bucket has previously been a target for CloudTrail log files, an IAM policy exists for the bucket. `UpdateTrail` must be called from the region in which the trail was created; otherwise, an `InvalidHomeRegionException` is thrown.

# Arguments

## `Name = ::String` -- *Required*
Specifies the name of the trail or trail ARN. If `Name` is a trail name, the string must meet the following requirements:

*   Contain only ASCII letters (a-z, A-Z), numbers (0-9), periods (.), underscores (_), or dashes (-)

*   Start with a letter or number, and end with a letter or number

*   Be between 3 and 128 characters

*   Have no adjacent periods, underscores or dashes. Names like `my-_namespace` and `my--namespace` are invalid.

*   Not be in IP address format (for example, 192.168.5.4)

If `Name` is a trail ARN, it must be in the format:

`arn:aws:cloudtrail:us-east-1:123456789012:trail/MyTrail`

## `S3BucketName = ::String`
Specifies the name of the Amazon S3 bucket designated for publishing log files. See [Amazon S3 Bucket Naming Requirements](http://docs.aws.amazon.com/awscloudtrail/latest/userguide/create_trail_naming_policy.html).

## `S3KeyPrefix = ::String`
Specifies the Amazon S3 key prefix that comes after the name of the bucket you have designated for log file delivery. For more information, see [Finding Your CloudTrail Log Files](http://docs.aws.amazon.com/awscloudtrail/latest/userguide/cloudtrail-find-log-files.html). The maximum length is 200 characters.

## `SnsTopicName = ::String`
Specifies the name of the Amazon SNS topic defined for notification of log file delivery. The maximum length is 256 characters.

## `IncludeGlobalServiceEvents = ::Bool`
Specifies whether the trail is publishing events from global services such as IAM to the log files.

## `IsMultiRegionTrail = ::Bool`
Specifies whether the trail applies only to the current region or to all regions. The default is false. If the trail exists only in the current region and this value is set to true, shadow trails (replications of the trail) will be created in the other regions. If the trail exists in all regions and this value is set to false, the trail will remain in the region where it was created, and its shadow trails in other regions will be deleted.

## `EnableLogFileValidation = ::Bool`
Specifies whether log file validation is enabled. The default is false.

**Note**
> When you disable log file integrity validation, the chain of digest files is broken after one hour. CloudTrail will not create digest files for log files that were delivered during a period in which log file integrity validation was disabled. For example, if you enable log file integrity validation at noon on January 1, disable it at noon on January 2, and re-enable it at noon on January 10, digest files will not be created for the log files delivered from noon on January 2 to noon on January 10. The same applies whenever you stop CloudTrail logging or delete a trail.

## `CloudWatchLogsLogGroupArn = ::String`
Specifies a log group name using an Amazon Resource Name (ARN), a unique identifier that represents the log group to which CloudTrail logs will be delivered. Not required unless you specify CloudWatchLogsRoleArn.

## `CloudWatchLogsRoleArn = ::String`
Specifies the role for the CloudWatch Logs endpoint to assume to write to a user's log group.

## `KmsKeyId = ::String`
Specifies the KMS key ID to use to encrypt the logs delivered by CloudTrail. The value can be an alias name prefixed by "alias/", a fully specified ARN to an alias, a fully specified ARN to a key, or a globally unique identifier.

Examples:

*   alias/MyAliasName

*   arn:aws:kms:us-east-1:123456789012:alias/MyAliasName

*   arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012

*   12345678-1234-1234-1234-123456789012

# Returns

`UpdateTrailResponse`

# Exceptions

`S3BucketDoesNotExistException`, `InsufficientS3BucketPolicyException`, `InsufficientSnsTopicPolicyException`, `InsufficientEncryptionPolicyException`, `TrailNotFoundException`, `InvalidS3BucketNameException`, `InvalidS3PrefixException`, `InvalidSnsTopicNameException`, `InvalidKmsKeyIdException`, `InvalidTrailNameException`, `TrailNotProvidedException`, `InvalidParameterCombinationException`, `InvalidHomeRegionException`, `KmsKeyNotFoundException`, `KmsKeyDisabledException`, `KmsException`, `InvalidCloudWatchLogsLogGroupArnException`, `InvalidCloudWatchLogsRoleArnException`, `CloudWatchLogsDeliveryUnavailableException`, `UnsupportedOperationException` or `OperationNotPermittedException`.

See also: [AWS API Documentation](https://docs.aws.amazon.com/goto/WebAPI/cloudtrail-2013-11-01/UpdateTrail)
"""
@inline update_trail(aws::AWSConfig=default_aws_config(); args...) = update_trail(aws, args)

@inline update_trail(aws::AWSConfig, args) = AWSCore.Services.cloudtrail(aws, "UpdateTrail", args)

@inline update_trail(args) = update_trail(default_aws_config(), args)

end # module CloudTrail

#==============================================================================#
# End of file
#==============================================================================#


text:
export train!, policy_evaluation!, policy_improvement!, policy_iteration!, value_iteration!

"""
    train!(env, agent; callbacks::Tuple{Vararg{<:Function}}=(stop_at_step(1),))

Train an `agent` in `env`.
By default `callbacks` is `(stop_at_step(1),)`. So it will only train one step forward.
"""
train!(env, agent; callbacks::Tuple{Vararg{<:Function}}=(stop_at_step(1),)) = train!(env, agent, buffertype(agent); callbacks=callbacks)

function train!(env::AbstractSyncEnvironment{Tss, Tas, 1} where {Tss, Tas},
                agent::AbstractAgent,
                ::Type{<:SARDBuffer};
                callbacks)
    obs, d = observe(env)
    if isempty(buffer(agent))
        s, a = agent(obs)  # TODO: check buffer first
        push!(buffer(agent), s, a)
    else
        s, a = buffer(agent).state[end], buffer(agent).action[end]
    end

    isstop = false
    while !isstop
        if d
            reset!(env)
            s, a = agent(observe(env).observation)
            push!(buffer(agent), s, a)
        end

        obs, r, d = env(a)  # TODO: split into two steps: 1. env(a), 2. observe(env)
        s, a = agent(obs)
        push!(buffer(agent), r, d, s, a)
        update!(agent)

        for cb in callbacks
            res = cb(env, agent)
            if res isa Bool && res
                isstop = true
            end
        end
    end
    callbacks
end

"""
    train!(env::AbstractSyncEnvironment{Tss, Tas, N} where {Tss, Tas},
           agents::Tuple{Vararg{<:Agent{<:AbstractLearner, <:SARDSBuffer}, N}};
           callbacks::Tuple{Vararg{<:Function}}=(stop_at_step(1),)) where N

TODO: Add an `AgentManager` struct to better organize `agents`.

For sync environments of mulit-agents, it becomes much more complicated compared to the single agent environments.
Here is an implementation for one of the most common cases. Each agent take an action alternately.
In every step, all agents may observe partial/complete information of the environment from their own perspective.

You may consider to overwrite this function according to the problem you want to solve.
"""
function train!(env::AbstractSyncEnvironment{Tss, Tas, N} where {Tss, Tas},
                agents::Tuple{Vararg{<:Agent{<:AbstractLearner, <:SARDBuffer}, N}};
                callbacks::Tuple{Vararg{<:Function}}=(stop_at_step(1),)) where N
    named_agents = Dict((agent.role, agent) for agent in agents)

    a = nothing
    next_role = get_next_role(env)

    for agent in agents
        obs = observe(env, agent.role).observation
        if agent.role == next_role
            s, a = agent(obs)
            push!(buffer(agent), s, a)
        else
            push!(buffer(agent), agent.preprocessor(obs), get_idle_action(env))
        end
    end

    isstop = false
    while !isstop
        next_role = get_next_role(env)
        if next_role === nothing
            reset!(env)
            next_role = get_next_role(env)
            for agent in agents
                obs = observe(env, agent.role).observation
                if agent.role == next_role
                    s, a = agent(obs)
                    push!(buffer(agent), s, a)
                else
                    push!(buffer(agent), agent.preprocessor(obs), get_idle_action(env))
                end
            end
        end

        env(a, next_role)  # now take action
        next_role = get_next_role(env)

        for agent in agents
            obs, isdone, reward = observe(env, agent.role)
            if agent.role == next_role
                s, a = agent(obs)
                push!(buffer(agent), reward, isdone, s, a)
            else
                push!(buffer(agent), reward, isdone, agent.preprocessor(obs), get_idle_action(env))
            end
            update!(agent)
        end

        for cb in callbacks
            res = cb(env, named_agents)
            if res isa Bool && res
                isstop = true
            end
        end
    end
    callbacks
end

"""
    policy_evaluation!(V::AbstractVApproximator, π::AbstractPolicy, Model::AbstractDistributionModel; γ::Float64=0.9, θ::Float64=1e-4)

See more details at Section (4.1) on Page 75 of the book *Sutton, Richard S., and Andrew G. Barto. Reinforcement learning: An introduction. MIT press, 2018.*
"""
function policy_evaluation!(V::AbstractVApproximator, π::AbstractPolicy, Model::AbstractDistributionModel; γ::Float64=0.9, θ::Float64=1e-4)
    states, actions = getstates(Model), getactions(Model)
    while true
        Δ = 0.
        for s in states
            v = sum(π(s, Val(:dist)) .* (sum(p * (r + γ * V(s′)) for (s′, r, p) in Model(s, a)) for a in actions))
            error = v - V(s)
            update!(V, s, error)
            Δ = max(Δ, abs(error))
        end
        Δ < θ && break
    end
end

"""
    policy_improvement!(V::AbstractVApproximator, π::AbstractPolicy, Model::AbstractDistributionModel; γ::Float64=0.9)

See more details at Section (4.2) on Page 76 of the book *Sutton, Richard S., and Andrew G. Barto. Reinforcement learning: An introduction. MIT press, 2018.*
"""
function policy_improvement!(V::AbstractVApproximator, π::AbstractPolicy, Model::AbstractDistributionModel; γ::Float64=0.9)
    states, actions = getstates(Model), getactions(Model)
    is_policy_stable = true
    for s in states
        old_a = π(s)
        best_action_inds = findallmax(sum(p * (r + γ * V(s′)) for (s′, r, p) in Model(s, a)) for a in actions)[2]
        new_a = actions[sample(best_action_inds)]
        if new_a != old_a
            update!(π, s, new_a)
            is_policy_stable = false
        end
    end
    is_policy_stable
end

"""
    policy_iteration!(V::AbstractVApproximator, π::AbstractPolicy, Model::AbstractDistributionModel; γ::Float64=0.9, θ::Float64=1e-4, max_iter=typemax(Int))

See more details at Section (4.3) on Page 80 of the book *Sutton, Richard S., and Andrew G. Barto. Reinforcement learning: An introduction. MIT press, 2018.*
"""
function policy_iteration!(V::AbstractVApproximator, π::AbstractPolicy, Model::AbstractDistributionModel; γ::Float64=0.9, θ::Float64=1e-4, max_iter=typemax(Int))
    for i in 1:max_iter
        @debug "iteration: $i"
        policy_evaluation!(V, π, Model; γ=γ, θ=θ)
        policy_improvement!(V, π, Model; γ=γ) && break
    end
end

"""
    value_iteration!(V::AbstractVApproximator, Model::AbstractDistributionModel; γ::Float64=0.9, θ::Float64=1e-4, max_iter=typemax(Int))

See more details at Section (4.4) on Page 83 of the book *Sutton, Richard S., and Andrew G. Barto. Reinforcement learning: An introduction. MIT press, 2018.*
"""
function value_iteration!(V::AbstractVApproximator, Model::AbstractDistributionModel; γ::Float64=0.9, θ::Float64=1e-4, max_iter=typemax(Int))
    states, actions = getstates(Model), getactions(Model)
    for i in 1:max_iter
        Δ = 0.
        for s in states
            v = maximum(sum(p * (r + γ * V(s′)) for (s′, r, p) in Model(s, a)) for a in actions)
            error = v - V(s)
            update!(V, s, error)
            Δ = max(Δ, abs(error))
        end
        Δ < θ && break
    end
end


text:
@testset "Plotting $f" for f in (:empty, :onetask, :chain, :shortlong, :medium)
    # Just making sure no errors are thrown
    proj = getfield(Examples, f)
    pert_chart = visualize_chart(proj)
end


text:
using LibCEED, Printf

include("common.jl")

function transform_mesh_coords!(dim, mesh_size, mesh_coords)
    @witharray coords = mesh_coords begin
        if dim == 1
            for i = 1:mesh_size
                # map [0,1] to [0,1] varying the mesh density
                coords[i] = 0.5 + 1.0/sqrt(3.0)*sin((2.0/3.0)*pi*(coords[i] - 0.5))
            end
            exact_volume = 1.0
        else
            num_nodes = mesh_size÷dim
            @inbounds @simd for i = 1:num_nodes
                # map (x,y) from [0,1]x[0,1] to the quarter annulus with polar
                # coordinates, (r,phi) in [1,2]x[0,pi/2] with area = 3/4*pi
                u = coords[i]
                v = coords[i+num_nodes]
                u = 1.0 + u
                v = pi/2*v
                coords[i] = u*cos(v)
                coords[i+num_nodes] = u*sin(v)
            end
            exact_volume = 3.0/4.0*pi
        end
        return exact_volume
    end
end

function run_ex1(; ceed_spec, dim, mesh_order, sol_order, num_qpts, prob_size, gallery)
    ncompx = dim
    prob_size < 0 && (prob_size = 256*1024)

    ceed = Ceed(ceed_spec)
    mesh_basis =
        create_tensor_h1_lagrange_basis(ceed, dim, ncompx, mesh_order + 1, num_qpts, GAUSS)
    sol_basis =
        create_tensor_h1_lagrange_basis(ceed, dim, 1, sol_order + 1, num_qpts, GAUSS)

    # Determine the mesh size based on the given approximate problem size.
    nxyz = get_cartesian_mesh_size(dim, sol_order, prob_size)
    println("Mesh size: ", nxyz)

    # Build CeedElemRestriction objects describing the mesh and solution discrete
    # representations.
    mesh_size, mesh_restr, _ =
        build_cartesian_restriction(ceed, dim, nxyz, mesh_order, ncompx, num_qpts)
    sol_size, sol_restr, sol_restr_i = build_cartesian_restriction(
        ceed,
        dim,
        nxyz,
        sol_order,
        1,
        num_qpts,
        mode=RestrictionAndStrided,
    )
    println("Number of mesh nodes     : ", div(mesh_size, dim))
    println("Number of solution nodes : ", sol_size)

    # Create a CeedVector with the mesh coordinates.
    mesh_coords = CeedVector(ceed, mesh_size)
    set_cartesian_mesh_coords!(dim, nxyz, mesh_order, mesh_coords)
    # Apply a transformation to the mesh.
    exact_vol = transform_mesh_coords!(dim, mesh_size, mesh_coords)

    # Create the Q-function that builds the mass operator (i.e. computes its
    # quadrature data) and set its context data.
    if !gallery
        @interior_qf build_qfunc = (
            ceed,
            dim=dim,
            (J, :in, EVAL_GRAD, dim, dim),
            (w, :in, EVAL_WEIGHT),
            (qdata, :out, EVAL_NONE),
            begin
                qdata .= w*det(J)
            end,
        )
    else
        build_qfunc = create_interior_qfunction(ceed, "Mass$(dim)DBuild")
    end

    # Create the operator that builds the quadrature data for the mass operator.
    build_oper = Operator(
        ceed,
        qf=build_qfunc,
        fields=[
            (gallery ? :dx : :J, mesh_restr, mesh_basis, CeedVectorActive()),
            (gallery ? :weights : :w, ElemRestrictionNone(), mesh_basis, CeedVectorNone()),
            (:qdata, sol_restr_i, BasisCollocated(), CeedVectorActive()),
        ],
    )

    # Compute the quadrature data for the mass operator.
    elem_qpts = num_qpts^dim
    num_elem = prod(nxyz)
    qdata = CeedVector(ceed, num_elem*elem_qpts)

    print("Computing the quadrature data for the mass operator ...")
    flush(stdout)
    apply!(build_oper, mesh_coords, qdata)
    println(" done.")

    # Create the Q-function that defines the action of the mass operator.
    if !gallery
        @interior_qf apply_qfunc = (
            ceed,
            (u, :in, EVAL_INTERP),
            (qdata, :in, EVAL_NONE),
            (v, :out, EVAL_INTERP),
            begin
                v .= qdata*u
            end,
        )
    else
        apply_qfunc = create_interior_qfunction(ceed, "MassApply")
    end

    # Create the mass operator.
    oper = Operator(
        ceed,
        qf=apply_qfunc,
        fields=[
            (:u, sol_restr, sol_basis, CeedVectorActive()),
            (:qdata, sol_restr_i, BasisCollocated(), qdata),
            (:v, sol_restr, sol_basis, CeedVectorActive()),
        ],
    )

    # Compute the mesh volume using the mass operator: vol = 1^T \cdot M \cdot 1
    print("Computing the mesh volume using the formula: vol = 1^T.M.1 ...")
    flush(stdout)
    # Create auxiliary solution-size vectors.
    u = CeedVector(ceed, sol_size)
    v = CeedVector(ceed, sol_size)
    # Initialize 'u' with ones.
    u[] = 1.0
    # Apply the mass operator: 'u' -> 'v'.
    apply!(oper, u, v)
    # Compute and print the sum of the entries of 'v' giving the mesh volume.
    vol = witharray_read(sum, v, MEM_HOST)

    println(" done.")
    @printf("Exact mesh volume    : % .14g\n", exact_vol)
    @printf("Computed mesh volume : % .14g\n", vol)
    @printf("Volume error         : % .14g\n", vol - exact_vol)
end

run_ex1(
    ceed_spec="/cpu/self",
    dim=3,
    mesh_order=4,
    sol_order=4,
    num_qpts=4 + 2,
    prob_size=-1,
    gallery=false,
)

max_stars_repo_name: AdelekeBankole/libCEED

max_issues_repo_name: AdelekeBankole/libCEED

max_forks_repo_name: AdelekeBankole/libCEED


text:
"Decompose a ColorTypes.RGB{N0f16} into red, green and blue color channels"
rgb_components(c::RGB{N0f16}) = red(c), green(c), blue(c)

"write tiffile from Cloud"
function to_tif(
    outDir::String,
    filenames::Vector{String},
    cloud::Cloud,
    cellsize::Real,
    epsg::Int;
    reducer = minz,
    pointfilter = nothing,
    min_dens = 0,
    nodata = -9999,
    return_density = false)

    # create grid definition
    r = define_raster(cloud, cellsize; epsg=epsg, pointfilter = pointfilter)

    to_tif(outDir, filenames, cloud, r;
        reducer = reducer, pointfilter = nothing, # filter in define_raster cellsize
        min_dens = min_dens, nodata = nodata, return_density = return_density)
end

"write tiffile from Cloud with given raster definition"
function to_tif(outDir::String, filenames::Vector{String},
    cloud::Cloud, r::Raster;
    reducer = minz, pointfilter = nothing,
    min_dens = 0, nodata = -9999, return_density = false)

    # make raster from Cloud based on statistical summary per griddcell and point filter
    grid = rasterize(cloud, r;
        reducer = reducer,
        pointfilter = pointfilter,
        min_dens = min_dens,
        nodata = nodata,
        return_density = return_density)

    # write to file
    grid2tif(outDir, filenames, r, grid;
        nodata=nodata)
end

"""util function to save a grid to tif using properties from raster defintion r
for 2d and 3d (multiple layers) grids; 3d grids are written to multiple files"""
function grid2tif(outDir::String, filenames::Vector{String}, r::Raster, grid::Array; nodata=-9999)
    ndim = ndims(grid)
    grid = copy(grid)
    if ndim == 3
        nlayers = size(grid, 3)
        for i in 1:nlayers
            fn = joinpath(outDir, filenames[i])
            grid[:, :, i][r.mask] = nodata
            write_raster(fn, grid[:, :, i], r.bbox.xmin, r.bbox.ymax, r.cellsize; epsg=XYZ.epsg(r), nodata=nodata)
        end
    elseif ndim == 2
        fn = joinpath(outDir, filenames[1])
        grid[r.mask] = nodata
        write_raster(fn, grid, r.bbox.xmin, r.bbox.ymax, r.cellsize; epsg=XYZ.epsg(r), nodata=nodata)
    end
end
"for 2d arrays"
function grid2tif(outDir::String, filename::String, r::Raster, grid::Array; nodata=-9999)
    @assert (ndims(grid) == 2) "Invalid grid size, use a vector of filenames to save layers to multiple files"
    fn = joinpath(outDir, filename)
    grid = copy(grid)
    grid[r.mask] = nodata
    write_raster(fn, grid, r.bbox.xmin, r.bbox.ymax, r.cellsize; epsg=XYZ.epsg(r), nodata=nodata)
end

"merge two headers, rejecting incompatible headers, and favoring the first"
function Base.merge(h1::LasHeader, h2::LasHeader)
    # check compatibility
    msg = "Cannot merge files, "
    h1.version_major === h2.version_major || error(msg * "LAS major version mismatch")
    h1.version_minor === h2.version_minor || error(msg * "LAS minor version mismatch")
    h1.data_format_id === h2.data_format_id || error(msg * "Point format mismatch")
    h1.data_format_id === h2.data_format_id || error(msg * "Point format mismatch")
    h1.data_record_length === h2.data_record_length || error(msg * "Point record length mismatch")

    # merge selected fields
    records_count = h1.records_count + h2.records_count
    point_return_count = h1.point_return_count + h2.point_return_count
    x_max = max(h1.x_max, h2.x_max)
    x_min = min(h1.x_min, h2.x_min)
    y_max = max(h1.y_max, h2.y_max)
    y_min = min(h1.y_min, h2.y_min)
    z_max = max(h1.z_max, h2.z_max)
    z_min = min(h1.z_min, h2.z_min)

    # Note that because this function is used for appending LAS files, the size of the header
    # must keep the same size. Therefore the VLRs are currently not extended, and neither are
    # the user defined bytes.
    # Furthermore it is assumed that the new coordinates will fit in with the existing h1 scale
    # and offset values.

    # create merged header
    LasHeader(
        h1.file_source_id,
        h1.global_encoding,
        h1.guid_1,
        h1.guid_2,
        h1.guid_3,
        h1.guid_4,
        h1.version_major,
        h1.version_minor,
        h1.system_id,
        h1.software_id,
        h1.creation_doy,
        h1.creation_year,
        h1.header_size,
        h1.data_offset,
        h1.n_vlr,
        h1.data_format_id,
        h1.data_record_length,
        records_count,
        point_return_count,
        h1.x_scale,
        h1.y_scale,
        h1.z_scale,
        h1.x_offset,
        h1.y_offset,
        h1.z_offset,
        x_max,
        x_min,
        y_max,
        y_min,
        z_max,
        z_min,
        h1.variable_length_records, # VLRs currently not extended, would need to be merged but not duplicated
        h1.user_defined_bytes
    )
end

"write LAS points"
function writepoints(io::IO, header::LasHeader, cloud::Cloud)
    n = length(positions(cloud))
    pointtype = LasIO.pointformat(header)
    for i in 1:n
        lasp = laspoint(pointtype, cloud, i, header)
        write(io, lasp)
    end
end

"Construct a LasIO Vector{LasPoint} from a XYZ.Cloud"
function lasio_vector(header::LasHeader, cloud::Cloud)
    n = length(positions(cloud))
    pointtype = LasIO.pointformat(header)
    pointdata = Vector{pointtype}(n)
    for i in 1:n
        pointdata[i] = laspoint(pointtype, cloud, i, header)
    end
    pointdata
end

"write LAS/LAZ file from Cloud"
function to_las(outDir::String, filename::String,
                cloud::Cloud, header::LasHeader)

    fn = joinpath(outDir, filename)

    header = deepcopy(header) # prevent unwanted side effects
    LasIO.update!(header, cloud)

    # converts to FileIO File LAS or LAZ
    # such that it dispatches to the right save
    fio = query(fn)
    save(fio, header, cloud)
    nothing
end

# Add save on Cloud be able to write LAS from Cloud without copies
function save(f::File{format"LAS"}, header::LasHeader, cloud::Cloud)
    # convert File to String to get a normal IO object
    # that the other functions can use
    fn = filename(f)
    open(fn, "w") do io
        write(io, "LASF")
        write(io, header)
        writepoints(io, header, cloud)
    end
end

# Add save on Cloud be able to write LAZ from Cloud without copies
function save(f::FileIO.File{FileIO.DataFormat{:LAZ}}, header::LasHeader, cloud::Cloud)
    # pipes las to laszip to write laz
    fn = filename(f)
    open(`laszip -olaz -stdin -o "$fn"`, "w") do s
        savebuf(s, header, cloud)
    end
end

# Add savebuf on Cloud be able to write LAZ from Cloud without copies
function savebuf(s::IO, header::LasHeader, cloud::Cloud)
    # checks
    header_n = header.records_count
    n = length(cloud.positions)
    msg = "number of records in header ($header_n) does not match data length ($n)"
    @assert header_n == n msg
    pointtype = LasIO.pointformat(header)

    # write header
    write(s, magic(format"LAS"))
    write(s, header)

    # 2048 points seemed to be an optimum for the libLAS_1.2.las testfile
    npoints_buffered = 2048
    bufsize = header.data_record_length * npoints_buffered
    buf = IOBuffer(bufsize)
    # write points
    for i in 1:n
        p = laspoint(pointtype, cloud, i, header)
        write(buf, p)
        if rem(i, npoints_buffered) == 0
            write(s, take!(buf))
        elseif i == n
            write(s, take!(buf))
        end
    end
end

"general XYZ (csv) writer"
function to_xyz(outDir::String, filename::String, cloud::Cloud;
    precision = 2, delimiter=",",
    attributes = Symbol[],
    pointfilter = nothing)

    fn = joinpath(outDir, filename)
    nrow = length(positions(cloud))
    write_attributes = length(attributes) > 0
    header =  xyz_header(attributes; delimiter=delimiter)

    # open file
    open(fn, "w") do f
        # write header
        write(f, header)

        # loop through points and write a line per point to file
        for i in 1:nrow
            (pointfilter != nothing) && (pointfilter(cloud, i) || continue)
            line = String[]
            # read and position point data and format
            for p in positions(cloud)[i]
                push!(line, format(p, precision=precision))
                push!(line, delimiter)
            end
            # read and attribute point data and format
            if write_attributes
                for key in attributes
                    if typeof(cloud[key][i]) <: Integer
                        push!(line, format(Int(cloud[key][i]), precision=0))
                    else
                        push!(line, format(cloud[key][i], precision=precision))
                    end
                    push!(line, delimiter)
                end
            end

          # write formatted line
          write(f, string(join(line)[1:end-length(delimiter)], "\n"))
        end
    end # close file
end

"XYZ (csv) writer for points along profile. The coordinates in profile orientation
are written in extra columns px and py to xyz file"
function to_xyz(outDir::String, filename::String, cloud::Cloud, prof::Profile;
    precision = 2, delimiter=",",
    attributes = Symbol[],
    pointfilter = nothing)

    fn = joinpath(outDir, filename)
    nrow = length(prof)
    write_attributes = length(attributes) > 0
    header = []
    header =  xyz_header(cat(1, ["px", "py"], attributes); delimiter=delimiter)

    # open file
    open(fn, "w") do f
        # write header
        write(f, header)

        # loop through points and write a line per point to file
        ip = 0
        for i in pointindex(prof)
            ip += 1
            (pointfilter != nothing) && (pointfilter(cloud, i) || continue)
            line = String[] # create vector with formated point data
            # read profile coordinates and format
            # read and position point data and format
            for p in positions(cloud)[i]
                push!(line, format(p, precision=precision))
                push!(line, delimiter)
            end
            pos = Float64[pointlx(prof)[ip], pointly(prof)[ip]]
            for p in pos
                push!(line, format(p, precision=precision))
                push!(line, delimiter)
            end
            # read and attribute point data and format
            if write_attributes
                for key in attributes
                    if typeof(cloud[key][i]) <: Integer
                        push!(line, format(Int(cloud[key][i]), precision=0))
                    else
                        push!(line, format(cloud[key][i], precision=precision))
                    end
                  push!(line, delimiter)
                end
            end

            # write formatted line
            write(f, string(join(line)[1:end-length(delimiter)], "\n"))
        end
    end # close file
end

"create header for xyz files"
function xyz_header(attributes::Vector{}; delimiter=",")
    header = ["x,","y,","z,"]
    if length(attributes) > 0
        for key in attributes
            push!(header, string(key))
            push!(header, delimiter)
        end
    end
    string(join(header)[1:end-length(delimiter)], "\n")
end


text:
module KernelsTests

using Test
using Gridap.Arrays
using Gridap.TensorValues
using LinearAlgebra
using BlockArrays

test_kernel(+,(3,2),5)

@test kernel_return_types((+,/),1,1) == (Int,Float64)

f = bcast(+)
a = rand(3,2)
b = 3
c = a .+ b
test_kernel(f,(a,b),c)

#test_kernel(1,(),1)
#test_kernel(1,(1,),1)
#
#test_kernel([1,2],(),[1,2])
#test_kernel([1,2],(1,),[1,2])

k = elem(-)
test_kernel(k,(1,),-1)
test_kernel(k,([1,2],),[-1,-2])
test_kernel(k,(1,2),-1)
test_kernel(k,(1.0,2),-1.0)
test_kernel(k,(1,2.0),-1.0)
test_kernel(k,([1,2],2),[-1,0])
test_kernel(k,(2,[1,2]),[1,0])
test_kernel(k,([3,4],[1,2]),[2,2])

k = contract(-)
test_kernel(k,([3,4],[1,2]),3-1+4-2)

f = bcast(⋅)
a = fill(TensorValue(2,0,0,0,2,0,0,0,2),2)
b = VectorValue(1,2,3)
c = zeros(VectorValue{3,Int},2)
broadcast!(⋅,c,a,b)
test_kernel(f,(a,b),c)

a = rand(3,4)
b = rand(4)
c = rand(3)
k = MulKernel()
test_kernel(k,(a,b),a*b)
k = MulAddKernel(2,3)
test_kernel(k,(a,b,c),2*a*b+3*c,≈)

a = rand(3,4)
b = rand(4)
k = MulKernel()
test_kernel(k,(a,b),a*b)

blocks = [ [1 2; 3 4], [5 6 7 8; 9 10 11 12; 13 14 15 16], [1 2 3 4; 5 6 7 8], [1 2 3; 4 5 6; 7 8 9] ]
blockids = [(1,1),(2,2),(1,2),(3,3)]
ax = (blockedrange([2,3,3]), blockedrange([2,4,3]))
a = BlockArrayCoo(blocks,blockids,ax)

blocks = [ 10*[1,2], 20*[1,2,3] ]
blockids = [(1,),(3,)]
axs = (blockedrange([2,4,3]),)
b = BlockArrayCoo(blocks,blockids,axs)
test_kernel(k,(a,b),a*b)

c = a*b
k = MulAddKernel(2,3)
test_kernel(k,(a,b,c),2*a*b+3*c)

#k = MulKernel()
#cache = kernel_cache(k,a,b)
#using BenchmarkTools
#@btime apply_kernel!($cache,$k,$a,$b)
#
#
#k = MulAddKernel(2,3)
#cache = kernel_cache(k,a,b,c)
#@btime apply_kernel!($cache,$k,$a,$b,$c)

end # module


text:
module Stats
    using Reexport
    @reexport using StatsBase
end


text:
module DataPreprocessing

using CSV, DataFrames, Plots
using RollingFunctions, StatsBase, ShiftedArrays
using Downloads: download

url = "https://archive.ics.uci.edu/ml/machine-learning-databases/housing/"
dataset = "housing.data"
datadict = "housing.names"

download("$(url)$(dataset)", dataset)
download("$(url)$(datadict)", datadict)

# The row data is split into lines and delimited by varying amounts of whitespace
rows = readlines(open(dataset))
rowdata = map(row -> join(split(row), ","), rows)
csvdata = write(open("housing.csv", "w"), join(rowdata, "\n"))

# According to the data dictionary, the column names are as follows
columns = [:CRIM, :ZN, :INDUS, :CHAS, :NOX, :RM, :AGE, :DIS, :RAD, :TAX, :PTRATIO, :B, :LSTAT, :MEDV]

# Read the data and add the column header
df = CSV.read("housing.csv", DataFrame; header=columns)

# Descriptive statistics
size(df)
describe(df)

# Identify nominal and continuous variables
nominal = [:CHAS, :RAD]
continuous = [:CRIM, :ZN, :INDUS, :NOX, :RM, :AGE, :DIS, :TAX, :PTRATIO, :B, :LSTAT]

# Examine the distributions of the nominal variables
# Make a table of the counts of the level

histogram(df[!,:RAD], bins=0:25)
histogram(df[!, :CHAS])

radcount = counts(df[!, :RAD])
chascount = counts(df[!, :CHAS])

# Make histograms for all of the nominal variables

plot(
    histogram(df[:, :CRIM]),
    histogram(df[:, :ZN]),
    histogram(df[:, :INDUS]),
    histogram(df[:, :NOX]),
    histogram(df[:, :RM]),
    histogram(df[:, :AGE]),
    histogram(df[:, :DIS]),
    histogram(df[:, :TAX]),
    histogram(df[:, :PTRATIO]),
    histogram(df[:, :B]),
    histogram(df[:, :LSTAT]),
    histogram(df[:, :MEDV])
)

# :MEDV has a strange spike at the value 50. Try removing any exactly equal to 50.

df = df[df[:, :MEDV] .!== 50.0, :]

# Calculate the correlation every variable has with :MEDV

correlation_matrix = mapreduce(permutedims, vcat, [[corkendall(df[:, column], df[:, variable]) for variable in columns] for column in columns])

# Display it as a heat map

heatmap(correlation_matrix, xlabel=columns, ylabel=columns)

# Make a dataframe
df1 = DataFrame(correlation_matrix, columns)
df1[!, :ROWLABEL] = columns

# Sort
df2 = sort(df1, :MEDV, rev=true)

# Linear regression
cols = [:CRIM, :ZN, :INDUS, :NOX, :RM, :AGE, :DIS, :TAX, :PTRATIO, :LSTAT, :MEDV]
y = df[!, :MEDV]
X_n = Dict(col => df[!, col] for col in cols)

models = Dict(
    :CRIM => lm(@formula(MEDV ~ 1 + CRIM), df),
    :ZN => lm(@formula(MEDV ~ 1 + ZN), df),
    :INDUS => lm(@formula(MEDV ~ 1 + INDUS), df),
    :NOX => lm(@formula(MEDV ~ 1 + NOX), df),
    :RM => lm(@formula(MEDV ~ 1 + RM), df),
    :AGE => lm(@formula(MEDV ~ 1 + AGE), df),
    :DIS => lm(@formula(MEDV ~ 1 + DIS), df),
    :TAX => lm(@formula(MEDV ~ 1 + TAX), df),
    :PTRATIO => lm(@formula(MEDV ~ 1 + PTRATIO), df),
    :LSTAT => lm(@formula(MEDV ~ 1 + LSTAT), df),
    :MEDV => lm(@formula(MEDV ~ 1 + MEDV), df),
)

plots = Dict(i => scatter(X_n[i], y, smooth=true, xlabel=i, legend=false)
            for (i, model) in models)
n
plot(values(plots)..., size=(2000, 1000))

# Feature Selection! Remove B, Convert ZN to categorical, Transform CRIM
# B removed

end # module

max_stars_repo_name: hairshirt/DataPreprocessing

max_issues_repo_name: hairshirt/DataPreprocessing

max_forks_repo_name: hairshirt/DataPreprocessing