@@ -0,0 +1,3174 @@
+/**%0A * Copyright (C) 2016 Hurence (bailet.thomas@gmail.com)%0A * %3Cp%3E%0A * Licensed under the Apache License, Version 2.0 (the %22License%22);%0A * you may not use this file except in compliance with the License.%0A * You may obtain a copy of the License at%0A * %3Cp%3E%0A * http://www.apache.org/licenses/LICENSE-2.0%0A * %3Cp%3E%0A * Unless required by applicable law or agreed to in writing, software%0A * distributed under the License is distributed on an %22AS IS%22 BASIS,%0A * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.%0A * See the License for the specific language governing permissions and%0A * limitations under the License.%0A */%0Apackage com.hurence.logisland.processor;%0A%0Aimport com.google.common.collect.Lists;%0Aimport com.hurence.logisland.annotation.documentation.CapabilityDescription;%0Aimport com.hurence.logisland.annotation.documentation.Tags;%0Aimport com.hurence.logisland.component.PropertyDescriptor;%0Aimport com.hurence.logisland.record.FieldDictionary;%0Aimport com.hurence.logisland.record.Record;%0Aimport com.hurence.logisland.validator.StandardValidators;%0Aimport org.slf4j.Logger;%0Aimport org.slf4j.LoggerFactory;%0A%0Aimport java.util.Collection;%0Aimport java.util.Collections;%0Aimport java.util.List;%0Aimport java.util.Map;%0Aimport java.util.concurrent.ConcurrentHashMap;%0Aimport java.util.function.Function;%0Aimport java.util.function.Predicate;%0Aimport java.util.stream.Collectors;%0A%0A@Tags(%7B%22record%22, %22fields%22, %22remove%22, %22delete%22%7D)%0A@CapabilityDescription(%22Keep only distinct records based on a given field%22)%0Apublic class SelectDistinctRecords extends AbstractProcessor %7B%0A%0A    private static final long serialVersionUID = -270933070438408174L;%0A%0A    private static final Logger logger = LoggerFactory.getLogger(SelectDistinctRecords.class);%0A%0A    public static final PropertyDescriptor FILTERING_FIELD = new PropertyDescriptor.Builder()%0A            .name(%22field.name%22)%0A            .description(%22the field to distinct records%22)%0A            .required(true)%0A            .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)%0A            .defaultValue(FieldDictionary.RECORD_ID)%0A            .build();%0A%0A%0A    public static %3CT%3E Predicate%3CT%3E distinctByKey(Function%3C? super T, Object%3E keyExtractor) %7B%0A        Map%3CObject, Boolean%3E map = new ConcurrentHashMap%3C%3E();%0A        return t -%3E map.putIfAbsent(keyExtractor.apply(t), Boolean.TRUE) == null;%0A    %7D%0A%0A    @Override%0A    public Collection%3CRecord%3E process(ProcessContext context, Collection%3CRecord%3E records) %7B%0A%0A        try %7B%0A            String fieldToFilterOn = context.getPropertyValue(FILTERING_FIELD).asString();%0A            records.stream()%0A                    .filter(distinctByKey(record -%3E record.getField(fieldToFilterOn)))%0A                    .collect(Collectors.toList());%0A%0A%0A        %7D catch (Exception ex) %7B%0A            logger.warn(%22issue while trying to remove field list %7B%7D :  %7B%7D%22,%0A                    context.getPropertyValue(FILTERING_FIELD).asString(),%0A                    ex.toString());%0A        %7D%0A%0A        return records;%0A    %7D%0A%0A%0A    @Override%0A    public List%3CPropertyDescriptor%3E getSupportedPropertyDescriptors() %7B%0A        return Collections.unmodifiableList(%0A                Lists.newArrayList(FILTERING_FIELD));%0A    %7D%0A%7D%0A

add selectdistinct processor