package com.github.rnewson.couchdb.lucene;

/**
 * Copyright 2009 Robert Newson
 * 
 * Licensed under the Apache License, Version 2.0 (the "License"); 
 * you may not use this file except in compliance with the License. 
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0 
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import static com.github.rnewson.couchdb.lucene.Utils.text;

import java.io.IOException;
import java.io.InputStream;
import java.io.Reader;

import org.apache.commons.io.IOUtils;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
import org.apache.lucene.document.Document;
import org.apache.tika.metadata.DublinCore;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.parser.AutoDetectParser;
import org.apache.tika.parser.ParsingReader;

import com.ibm.icu.text.CharsetDetector;

public final class Tika {

	private static final Logger log = LogManager.getLogger(Tika.class);

	private static final String DC = "dc.";

	public void parse(final InputStream in, final String contentType, final Document doc) throws IOException {
		final AutoDetectParser parser = new AutoDetectParser();
		final Metadata md = new Metadata();
		md.set(Metadata.CONTENT_TYPE, contentType);

		final Reader reader = new ParsingReader(parser, in, md);
		final String body;
		try {
			try {
				body = IOUtils.toString(reader);
			} finally {
				reader.close();
			}
		} catch (final IOException e) {
			log.warn("Failed to index an attachment.", e);
			return;
		}

		// Add body text.
		doc.add(text(Config.BODY, body, false));
		// Add DC attributes.
		addDublinCoreAttributes(md, doc);
	}

	private void addDublinCoreAttributes(final Metadata md, final Document doc) {
		addAttribute(DC, DublinCore.CONTRIBUTOR, md, doc);
		addAttribute(DC, DublinCore.COVERAGE, md, doc);
		addAttribute(DC, DublinCore.CREATOR, md, doc);
		addAttribute(DC, DublinCore.DATE, md, doc);
		addAttribute(DC, DublinCore.DESCRIPTION, md, doc);
		addAttribute(DC, DublinCore.FORMAT, md, doc);
		addAttribute(DC, DublinCore.IDENTIFIER, md, doc);
		addAttribute(DC, DublinCore.LANGUAGE, md, doc);
		addAttribute(DC, DublinCore.MODIFIED, md, doc);
		addAttribute(DC, DublinCore.PUBLISHER, md, doc);
		addAttribute(DC, DublinCore.RELATION, md, doc);
		addAttribute(DC, DublinCore.RIGHTS, md, doc);
		addAttribute(DC, DublinCore.SOURCE, md, doc);
		addAttribute(DC, DublinCore.SUBJECT, md, doc);
		addAttribute(DC, DublinCore.TITLE, md, doc);
		addAttribute(DC, DublinCore.TYPE, md, doc);
	}
	
	private void addAttribute(final String namespace, final String attributeName, final Metadata md, final Document doc) {
		if (md.get(attributeName) != null) {
			doc.add(text(namespace + attributeName, md.get(attributeName), false));
		}
	}
}

@@ -927,16 +927,74 @@
 cument;%0A
+import org.apache.nutch.analysis.lang.LanguageIdentifier;%0A
 import o
@@ -1029,16 +1029,16 @@
 inCore;%0A
-
 import o
@@ -1169,50 +1169,8 @@
 r;%0A%0A
-import com.ibm.icu.text.CharsetDetector;%0A%0A
 publ
@@ -1304,16 +1304,98 @@
 %22dc.%22;%0A%0A
+%09private final LanguageIdentifier languageIdentifier = new LanguageIdentifier();%0A%0A
 %09public 
@@ -1974,16 +1974,17 @@
 alse));%0A
+%0A
 %09%09// Add
@@ -2030,24 +2030,132 @@
 es(md, doc);
+%0A%0A%09%09// Detect language.%0A%09%09doc.add(text(DC + DublinCore.LANGUAGE, languageIdentifier.identify(body), false));
 %0A%09%7D%0A%0A%09privat
@@ -2571,32 +2571,35 @@
 ER, md, doc);%0A%09%09
+// 
 addAttribute(DC,
@@ -3022,17 +3022,16 @@
 oc);%0A%09%7D%0A
-%09
 %0A%09privat

use Nutch's version of language.