/*
 * Copyright 2010 Internet Archive
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you
 * may not use this file except in compliance with the License. You
 * may obtain a copy of the License at
 *
 *  http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
 * implied. See the License for the specific language governing
 * permissions and limitations under the License.
 */

import java.io.*;
import java.net.*;
import java.util.*;

import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.client.solrj.*;
import org.apache.solr.client.solrj.impl.CommonsHttpSolrServer;

/**
 * 
 */
public class SolrDocumentWriter extends DocumentWriterBase
{
  private SolrServer server;

  public SolrDocumentWriter( URL url )
    throws IOException
  {
    this.server = new CommonsHttpSolrServer( url );
  }

  public void add( String key, DocumentProperties properties )
    throws IOException
  {
    // TODO: Create Solr XML document, the post it.
    SolrInputDocument doc = new SolrInputDocument();

    doc.addField( "id",  key );
    
    for ( String p : new String[] { "url", "title", "length", "collection", "boiled", "site", "type" } )
      {
        doc.addField( p, properties.get( p ) );
      }

    doc.addField( "content", properties.get( "content_parsed" ) );

    // Solr requires the date to be in the form: 1995-12-31T23:59:59Z
    // See the Solr schema docs.
    String date = properties.get( "date" );
    if ( date.length() == "yyyymmddhhmmss".length() )
      {
        doc.addField( "date", date.substring(0,4)  + "-" + date.substring(4,6)   + "-" + date.substring(6,8)   + "T" + 
                              date.substring(8,10) + ":" + date.substring(10,12) + ":" + date.substring(12,14) + "Z" );
      }

    try
      {
        this.server.add( doc );
      }
    catch ( SolrServerException sse )
      {
        throw new IOException( sse );
      }
  }

  public void commit( )
    throws IOException
  {
    try
      {
        this.server.commit();
      }
    catch ( SolrServerException sse )
      {
        throw new IOException( sse );
      }
  }
}
@@ -1350,16 +1350,8 @@
 ed%22,
- %22site%22,
  %22ty
@@ -1358,16 +1358,16 @@
 pe%22 %7D )%0A
+
       %7B%0A
@@ -1945,24 +1945,1058 @@
 );%0A      %7D%0A%0A
+    // Special handling for site%0A    try%0A      %7B%0A        String url = properties.get( %22url%22 );%0A        %0A        String site = (new URL( url)).getHost( );%0A        %0A        // Strip off any %22www%5B0-9%5D*.%22 header.%0A        site = site.toLowerCase().replaceFirst( %22%5Ewww%5B0-9%5D*%5B.%5D%22, %22%22 );%0A        %0A        // Special rule for Photobucket%0A        site = site.replaceAll( %22%5E%5Ba-z0-9%5D+.photobucket.com$%22, %22photobucket.com%22 );%0A        %0A        doc.addField( %22site%22, site );%0A%0A        // Add the tld: com, org, net, uk, fr, etc.%0A        if ( site.length() %3E 0 %0A             && site.charAt(0) != '%5B'                             // Not an IPv6 address%0A             && ! site.matches(%22(?:%5B0-9%5D%7B1,3%7D%5B.%5D)%7B3%7D%5B0-9%5D%7B1,3%7D%22)  // Not an IPv4 address%0A             )%0A          %7B%0A            String%5B%5D s = site.replaceFirst(%22%5B.%5D$%22, %22%22).split( %22%5B.%5D%22 );%0A%0A            String tld = s%5Bs.length-1%5D;%0A%0A            doc.addField( %22tld%22, tld );%0A          %7D%0A      %7D%0A    catch ( MalformedURLException mue )%0A      %7B%0A        // Rut-roh.%0A      %7D%0A%0A    // Finally, add the document.%0A
     try%0A    

Add special handling for site and tld.