@@ -0,0 +1,2599 @@
+/*%0A * Copyright 2015 OpenCB%0A *%0A * Licensed under the Apache License, Version 2.0 (the %22License%22);%0A * you may not use this file except in compliance with the License.%0A * You may obtain a copy of the License at%0A *%0A *     http://www.apache.org/licenses/LICENSE-2.0%0A *%0A * Unless required by applicable law or agreed to in writing, software%0A * distributed under the License is distributed on an %22AS IS%22 BASIS,%0A * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.%0A * See the License for the specific language governing permissions and%0A * limitations under the License.%0A */%0A%0Apackage org.opencb.hpg.bigdata.core.parquet;%0A%0Aimport org.apache.avro.Schema;%0Aimport org.apache.avro.file.DataFileStream;%0Aimport org.apache.avro.generic.GenericDatumReader;%0Aimport org.apache.avro.generic.IndexedRecord;%0Aimport org.apache.avro.io.DatumReader;%0Aimport org.apache.hadoop.fs.Path;%0Aimport org.apache.parquet.avro.AvroParquetWriter;%0Aimport org.apache.parquet.hadoop.ParquetWriter;%0Aimport org.apache.parquet.hadoop.metadata.CompressionCodecName;%0A%0Aimport java.io.IOException;%0Aimport java.io.InputStream;%0A%0A/**%0A * Created by hpccoll1 on 05/05/15.%0A */%0Apublic abstract class ParquetConverter%3CT extends IndexedRecord%3E %7B%0A%0A    protected CompressionCodecName compressionCodecName;%0A    protected int rowGroupSize;%0A    protected int pageSize;%0A%0A    protected Schema schema;%0A%0A    public ParquetConverter() %7B%0A        compressionCodecName = CompressionCodecName.GZIP;%0A        rowGroupSize = ParquetWriter.DEFAULT_BLOCK_SIZE;%0A        pageSize = ParquetWriter.DEFAULT_PAGE_SIZE;%0A    %7D%0A%0A    public ParquetConverter(Schema schema) %7B%0A        this();%0A%0A        this.schema = schema;%0A    %7D%0A%0A    public void toParquet(InputStream inputStream, String outputFile) throws IOException %7B%0A        DatumReader%3CT%3E datumReader = new GenericDatumReader%3C%3E(schema);%0A        DataFileStream%3CT%3E dataFileStream = new DataFileStream%3C%3E(inputStream, datumReader);%0A%0A        ParquetWriter%3CObject%3E parquetWriter = AvroParquetWriter.builder(new Path(outputFile))%0A                .withSchema(schema)%0A                .withCompressionCodec(compressionCodecName)%0A                .withRowGroupSize(rowGroupSize)%0A                .withPageSize(pageSize)%0A                .build();%0A%0A        int numRecords = 0;%0A        T record = null;%0A        while (dataFileStream.hasNext()) %7B%0A            record = dataFileStream.next(record);%0A                parquetWriter.write(record);%0A%0A            if (numRecords %25 10000 == 0) %7B%0A                System.out.println(numRecords);%0A            %7D%0A            numRecords++;%0A        %7D%0A%0A        parquetWriter.close();%0A    %7D%0A%0A%7D%0A

move ParquetConverter to core module from tools