/*
 * Copyright 2010 Internet Archive
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you
 * may not use this file except in compliance with the License. You
 * may obtain a copy of the License at
 *
 *  http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
 * implied. See the License for the specific language governing
 * permissions and limitations under the License.
 */

import java.io.*;
import java.net.*;
import java.util.*;

import org.apache.lucene.document.*;
import org.apache.lucene.index.*;

/**
 * Custom FieldHandler implementation for site.
 *
 * The site field not stored and it is indexed as a single token.  In
 * addition, we apply some very rudimentary canonicalization, such as
 * stripping a leading 'www[0-9]' and a special rule for Photobucket.
 *
 * Ideally we would apply more sophisticated rules, perhaps even
 * collection-specific, to better determine what the "site" is for a
 * URL.
 */ 
public class SiteHandler implements FieldHandler
{
  IDNHelper helper;

  public SiteHandler( )
  {
    this( new IDNHelper( ) );
  }

  public SiteHandler( IDNHelper helper )
  {
    this.helper = helper;
  }

  public void handle( Document doc, DocumentProperties properties )
  {
    // Special handling for site
    try
      {
        URL u = new URL( properties.get( "url" ) );

        String domain = this.helper.getDomain( u );

        // If we cannot determine the domain, use the full hostname.
        // This can happen if the URL uses IP address rather than
        // hostname.
        if ( domain == null ) 
          {
            domain = u.getHost( );
          }
        else
          {
            domain = IDN.toUnicode( domain, IDN.ALLOW_UNASSIGNED );
          }

        doc.add( new Field( "site", domain, Field.Store.NO, Field.Index.NOT_ANALYZED_NO_NORMS) );
      }
    catch ( MalformedURLException mue )
      {
        // Rut-roh.
      }
  }

}

@@ -845,292 +845,76 @@
 ken.
-  In%0A * addition, we apply some very rudimentary canonicalization, such as%0A * stripping a leading 'www%5B0-9%5D' and a special rule for Photobucket.%0A *%0A * Ideally we would apply more sophisticated rules, perhaps even%0A * collection-specific, to better determine what the %22site%22 is for a%0A *
+%0A *%0A * The IDNHelper is used to determine the domain of the%0A * given
  URL
@@ -1207,41 +1207,8 @@
   %7B%0A
-    // Special handling for site%0A
     
@@ -1846,15 +1846,110 @@
  // 
-Rut-roh
+Very strange for the URL of a crawled page to be malformed.%0A        // But, in that case, just skip it
 .%0A  

Update class description to include IDNHelper. Also clean-up a few other comments. No code changes.