@@ -0,0 +1,1692 @@
+from __future__ import print_function, division, absolute_import%0A%0Afrom into import discover, CSV, resource, convert%0Aimport pandas as pd%0Afrom ..utils import cls_name%0Afrom toolz import memoize%0Afrom datashape import var%0A%0A%0Aclass S3(object):%0A    %22%22%22An object that holds a resource that lives in an S3 bucket%0A%0A    Examples%0A    --------%0A    %3E%3E%3E bucket = S3(CSV('s3://nyqpug/tips.csv'))%0A    %3E%3E%3E dshape = discover(bucket)%0A    %3E%3E%3E dshape.measure%5B'total_bill'%5D%0A    ctype(%22float64%22)%0A%0A    Notes%0A    -----%0A    * pandas read_csv can read CSV files directly from S3 buckets%0A    * We should check to make sure that we are discovering from CSVs as fast as%0A      as possible, since using S3 requires hitting the network.%0A    * For more complicated formats we'll have to write some middleware that%0A      turns S3 blobs into a format suitable to be converted into other formats%0A      such as JSON, HDF5, etc. Some of this will be handled by the into graph.%0A    %22%22%22%0A    def __init__(self, data):%0A        self.data = self.container(data)%0A%0A%0A@memoize%0Adef s3(cls):%0A    %22%22%22Parametrized S3 bucket Class%0A%0A    Notes%0A    -----%0A    * Shamelessly copied from %60%60into.chunks%60%60%0A    %22%22%22%0A    return type('S3_' + cls_name(cls).replace('.', '_'), (S3,),%0A                %7B'container': cls%7D)%0A%0A%0A@resource.register('s3://.*%5C.csv', priority=18)%0Adef resource_s3_csv(uri):%0A    return s3(CSV)(uri)%0A%0A%0A@convert.register(CSV, s3(CSV))%0Adef convert_s3_to_csv(bucket, **kwargs):%0A    return CSV(bucket.data.path, **kwargs)%0A%0A%0A@discover.register(s3(CSV))%0Adef discover_s3_csv(c, **kwargs):%0A    return var * discover(c.data).subshape%5B0%5D%0A%0A%0A@convert.register(pd.DataFrame, s3(CSV))%0Adef convert_csv_to_s3(csv):%0A    return convert(pd.DataFrame, csv.data)%0A

Implement part of the s3 csv into interface