/*
 * Copyright 2010 Internet Archive
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you
 * may not use this file except in compliance with the License. You
 * may obtain a copy of the License at
 *
 *  http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
 * implied. See the License for the specific language governing
 * permissions and limitations under the License.
 */

import org.apache.lucene.search.DefaultSimilarity;

/** 
 * Lucene Similarity implementatation appropriate for web searching.
 * Intitially, taken from NutchSimilarity, then tweaked.
 */ 
public class WebSimilarity extends DefaultSimilarity  
{
  private static final int MIN_CONTENT_LENGTH = 1000;
  
  /** Normalize field by length.  Called at index time. */
  public float lengthNorm( String fieldName, int numTokens )
  {
    if ("url".equals(fieldName))
      {
        // URL: prefer short by using linear normalization
        return 1.0f / numTokens;
        
      }
    else if ("anchor".equals(fieldName))
      { 
        // Anchor: prefer more
        return (float)(1.0/Math.log(Math.E+numTokens)); // use log
        
      }
    else if ("content".equals(fieldName))
      {    
        // Content: penalize short, by treating short as longer
        return super.lengthNorm( fieldName, Math.max(numTokens, MIN_CONTENT_LENGTH) );
      }
    else
      {
        // use default
        return super.lengthNorm(fieldName, numTokens);
      }
  }
  
  public float coord(int overlap, int maxOverlap)
  {
    return 1.0f;
  }

}

@@ -1,56 +1,393 @@
 /*
+*
 %0A * 
-Copyright 2010 Internet Archive%0A *%0A * Licensed
+This code is derived from the class NutchSimilarity distributed%0A * with Nutch 1.1.  The original license is as follows:%0A *%0A *   Licensed to the Apache Software Foundation (ASF) under one or%0A *   more contributor license agreements.  See the NOTICE file%0A *   distributed with this work for additional information regarding%0A *   copyright ownership.  The ASF licenses this file to You
  und
@@ -388,24 +388,29 @@
 ou under the
+%0A *  
  Apache Lice
@@ -446,19 +446,16 @@
 e%22); you
-%0A *
  may not
@@ -459,24 +459,29 @@
 not use this
+%0A *  
  file except
@@ -516,15 +516,13 @@
 nse.
+ 
  You
-%0A *
  may
@@ -530,16 +530,21 @@
 obtain a
+%0A *  
  copy of
@@ -564,16 +564,19 @@
 at%0A *%0A *
+   
   http:/
@@ -617,16 +617,18 @@
 .0%0A *%0A *
+  
  Unless 
@@ -678,16 +678,21 @@
 writing,
+%0A *  
  softwar
@@ -692,19 +692,16 @@
 software
-%0A *
  distrib
@@ -747,16 +747,21 @@
 n an %22AS
+%0A *  
  IS%22 BAS
@@ -763,19 +763,16 @@
 %22 BASIS,
-%0A *
  WITHOUT
@@ -816,16 +816,21 @@
 , either
+%0A *  
  express
@@ -832,19 +832,16 @@
 press or
-%0A *
  implied
@@ -841,16 +841,17 @@
 implied.
+ 
  See the
@@ -884,16 +884,21 @@
 language
+%0A *  
  governi
@@ -899,19 +899,16 @@
 overning
-%0A *
  permiss
@@ -952,16 +952,65 @@
 e.%0A */%0A%0A
+import org.apache.lucene.index.FieldInvertState;%0A
 import o
@@ -1064,15 +1064,8 @@
  %0A *
- Lucene
  Sim
@@ -1370,22 +1370,23 @@
 c float 
-length
+compute
 Norm( St
@@ -1405,27 +1405,81 @@
 me, 
-int numTokens )%0A  %7B
+FieldInvertState state )%0A  %7B%0A    int numTokens = state.getLength( );%0A    
 %0A   
@@ -1643,14 +1643,15 @@
 f (%22
-anchor
+content
 %22.eq
@@ -1675,16 +1675,19 @@
       %7B 
+   
 %0A       
@@ -1694,229 +1694,181 @@
  // 
-Anchor: prefer more%0A        return (float)(1.0/Math.log(Math.E+numTokens)); // use log%0A        %0A      %7D%0A    else if (%22content%22.equals(fieldName))%0A      %7B    %0A        // Content: penalize short, by treating short as longer
+Content: penalize short, by treating short as longer%0A%0A        // TODO: Is creating a new FieldInvertState object good, or%0A        //       should we modify the existing one?
 %0A   
@@ -1881,30 +1881,31 @@
 eturn super.
-length
+compute
 Norm( fieldN
@@ -1913,46 +1913,485 @@
 me, 
-Math.max(numTokens, MIN_CONTENT_LENGTH
+new FieldInvertState( state.getPosition(),%0A                                                                   Math.max(numTokens, MIN_CONTENT_LENGTH),%0A                                                                   state.getNumOverlap(),%0A                                                                   state.getOffset(),%0A                                                                   state.getBoost() %0A                                                                 
 ) );
@@ -2460,22 +2460,23 @@
 n super.
-length
+compute
 Norm(fie
@@ -2483,25 +2483,21 @@
 ldName, 
-numTokens
+state
 );%0A     

Change over-ride of lengthNorm() to computeNorm() due to change in Lucene 3.0.1 API.