/*
 * Copyright 2016 Needham Software LLC
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.jesterj.ingest.processors;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.tika.Tika;
import org.apache.tika.exception.TikaException;
import org.apache.tika.metadata.Metadata;
import org.jesterj.ingest.model.Document;
import org.jesterj.ingest.model.DocumentProcessor;
import org.jesterj.ingest.model.Status;
import org.jesterj.ingest.model.impl.NamedBuilder;

import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.nio.charset.Charset;
import java.security.AccessControlException;

/*
 * Created with IntelliJ IDEA.
 * User: gus
 * Date: 3/19/16
 */
public class TikaProcessor implements DocumentProcessor {

  private static final Logger log = LogManager.getLogger();
  private String name;

  @Override
  public Document[] processDocument(Document document) {
    byte[] rawData = document.getRawData();
    try {
      Tika tika = new Tika();
      Metadata metadata = new Metadata();
      try (ByteArrayInputStream bais = new ByteArrayInputStream(rawData)) {
        String textContent = tika.parseToString(bais, metadata);
        document.setRawData(textContent.getBytes(Charset.forName("UTF-8")));
        for (String name : metadata.names()) {
          document.put(name, metadata.get(name));
        }
      } catch (IOException | TikaException e) {
        log.warn("Tika processing failure!", e);
        // if tika can't parse it we certainly don't want random binary crap in the index
        document.setStatus(Status.DROPPED);
      }
    } catch (Throwable t) {
      boolean isAccessControl = t instanceof AccessControlException;
      boolean isSecurity = t instanceof SecurityException;
      if (!isAccessControl && !isSecurity) {
        throw t;
      } else {
        System.out.println("gotcha!");
      }
    }
    return new Document[]{document};
  }

  @Override
  public String getName() {
    return name;
  }

  public static class Builder extends NamedBuilder<TikaProcessor> {

    TikaProcessor obj = new TikaProcessor();

    protected TikaProcessor getObj() {
      return obj;
    }

    public Builder named(String name) {
      getObj().name = name;
      return this;
    }

    private void setObj(TikaProcessor obj) {
      this.obj = obj;
    }

    public TikaProcessor build() {
      TikaProcessor object = getObj();
      setObj(new TikaProcessor());
      return object;
    }

  }

}

@@ -1379,16 +1379,41 @@
 ng name;
+%0A  private String suffix;
 %0A%0A  @Ove
@@ -1527,41 +1527,241 @@
     
-try %7B%0A      Tika tika = new Tika(
+if (rawData == null) %7B%0A      log.debug(%22Skipping document without data in %22 + getName());%0A      return new Document%5B%5D%7Bdocument%7D;%0A    %7D%0A    try %7B%0A      Tika tika = new Tika();%0A      tika.setMaxStringLength(document.getRawData().length
 );%0A 
@@ -2089,20 +2089,45 @@
 ent.put(
-name
+sanitize(name) + plusSuffix()
 , metada
@@ -2718,16 +2718,418 @@
 %7D;%0A  %7D%0A%0A
+  private String plusSuffix() %7B%0A    return suffix == null ? %22%22 : suffix;%0A  %7D%0A%0A  private String sanitize(String dirty) %7B%0A    StringBuilder clean = new StringBuilder(dirty.length());%0A    for (char c : dirty.toCharArray()) %7B%0A      if ((c %3E= 'a' && c %3C= 'z') %7C%7C (c %3E= 'A' && c %3C= 'Z')) %7B%0A        clean.append(c);%0A      %7D else %7B%0A        clean.append('_');%0A      %7D%0A    %7D%0A    return clean.toString();%0A  %7D%0A  %0A%0A
   @Overr
@@ -3451,24 +3451,134 @@
 his;%0A    %7D%0A%0A
+    public Builder appendingSuffix(String suffix) %7B%0A      getObj().suffix = suffix;%0A      return this;%0A    %7D%0A%0A
     private 

Fix #49 - suffixes and field name santization for Tika