/*
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package io.trino.orc;

import io.airlift.slice.Slices;
import io.airlift.units.DataSize;
import io.trino.orc.metadata.CompressionKind;
import io.trino.orc.metadata.statistics.NoOpBloomFilterBuilder;
import io.trino.orc.metadata.statistics.StringStatisticsBuilder;
import io.trino.orc.writer.SliceDictionaryColumnWriter;
import io.trino.spi.block.Block;
import io.trino.spi.block.RunLengthEncodedBlock;
import org.testng.annotations.Test;

import java.util.concurrent.ThreadLocalRandom;

import static io.airlift.units.DataSize.Unit.MEGABYTE;
import static io.trino.orc.OrcWriterOptions.DEFAULT_MAX_COMPRESSION_BUFFER_SIZE;
import static io.trino.orc.OrcWriterOptions.DEFAULT_MAX_STRING_STATISTICS_LIMIT;
import static io.trino.orc.metadata.OrcColumnId.ROOT_COLUMN;
import static io.trino.spi.type.VarcharType.VARCHAR;
import static java.lang.Math.toIntExact;
import static org.testng.Assert.assertFalse;

public class TestSliceDictionaryColumnWriter
{
    @Test
    public void testDirectConversion()
    {
        SliceDictionaryColumnWriter writer = new SliceDictionaryColumnWriter(
                ROOT_COLUMN,
                VARCHAR,
                CompressionKind.NONE,
                toIntExact(DEFAULT_MAX_COMPRESSION_BUFFER_SIZE.toBytes()),
                () -> new StringStatisticsBuilder(toIntExact(DEFAULT_MAX_STRING_STATISTICS_LIMIT.toBytes()), new NoOpBloomFilterBuilder()));

        // a single row group exceeds 2G after direct conversion
        byte[] value = new byte[megabytes(1)];
        ThreadLocalRandom.current().nextBytes(value);
        Block data = RunLengthEncodedBlock.create(VARCHAR, Slices.wrappedBuffer(value), 3000);
        writer.beginRowGroup();
        writer.writeBlock(data);
        writer.finishRowGroup();

        assertFalse(writer.tryConvertToDirect(megabytes(64)).isPresent());
    }

    private static int megabytes(int size)
    {
        return toIntExact(DataSize.of(size, MEGABYTE).toBytes());
    }
}

@@ -578,16 +578,47 @@
 o.orc;%0A%0A
+import io.airlift.slice.Slice;%0A
 import i
@@ -1072,16 +1072,160 @@
 Random;%0A
+import java.util.stream.IntStream;%0A%0Aimport static io.airlift.slice.SliceUtf8.lengthOfCodePoint;%0Aimport static io.airlift.slice.Slices.utf8Slice;
 %0Aimport 
@@ -1548,16 +1548,66 @@
 ARCHAR;%0A
+import static java.lang.Character.MAX_CODE_POINT;%0A
 import s
@@ -2186,163 +2186,231 @@
     
-// a single row group exceeds 2G after direct conversion%0A        byte%5B%5D value = new byte%5Bmegabytes(1)%5D;%0A        ThreadLocalRandom.current().nextBytes(value
+int%5B%5D validCodepoints = IntStream.range(0, MAX_CODE_POINT)%0A                .filter(Character::isValidCodePoint)%0A                .toArray();%0A        Slice randomUtf8Slice = createRandomUtf8Slice(validCodepoints, megabytes(1)
 );%0A 
@@ -2492,13 +2492,35 @@
 fer(
-value
+randomUtf8Slice.byteArray()
 ), 3
@@ -2706,16 +2706,674 @@
 %0A    %7D%0A%0A
+    public static Slice createRandomUtf8Slice(int%5B%5D codePointSet, int approximateLengthInBytes)%0A    %7B%0A        approximateLengthInBytes += 64;%0A        int%5B%5D codePoints = new int%5BapproximateLengthInBytes%5D;%0A        ThreadLocalRandom random = ThreadLocalRandom.current();%0A        int totalLength = 0;%0A        int offset = 0;%0A        while (totalLength %3C approximateLengthInBytes) %7B%0A            int codePoint = codePointSet%5Brandom.nextInt(codePointSet.length)%5D;%0A            codePoints%5Boffset%5D = codePoint;%0A            totalLength += lengthOfCodePoint(codePoint);%0A            offset++;%0A        %7D%0A        return utf8Slice(new String(codePoints, 0, offset));%0A    %7D%0A%0A
     priv

Generate only valid codepoints