/*
 * Copyright 2010 Internet Archive
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you
 * may not use this file except in compliance with the License. You
 * may obtain a copy of the License at
 *
 *  http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
 * implied. See the License for the specific language governing
 * permissions and limitations under the License.
 */

package org.archive.tnh.tools;

import java.io.IOException;
import java.io.File;

import org.apache.lucene.analysis.*;
import org.apache.lucene.index.*;
import org.apache.lucene.store.*;


public class IndexMerger
{
  public static void main( String[] args )
    throws Exception
  {
    if ( args.length < 2 )
      {
        System.err.println( "IndexMerger [-o|-m] <dest> <source>..." );
        System.exit( 1 );
      }

    boolean optimize = false;
    boolean merge    = false;

    int i = 0;
    for ( ; i < args.length ; i++ )
      {
        if ( "-o".equals( args[i] ) )
          {
            optimize = true;
          }
        else if ( "-m".equals( args[i] ) )
          {
            merge = true;
          }
        else
          {
            break ;
          }
      }

    File dest = new File( args[i++] );

    if ( ! merge && dest.exists( ) )
      {
        System.err.println( "Destination exits, use -m to merge.  Dest: " + args[i] );

        System.exit( 2 );
      }
    
    Directory d[] = new Directory[args.length-i];
    for ( int j = i ; j < args.length ; j++ )
      {
        d[j-i] = new NIOFSDirectory( new File( args[j] ) );
      }

    IndexWriter w = null;
    try
      {
        w = new IndexWriter( new NIOFSDirectory( dest ), null, true, IndexWriter.MaxFieldLength.UNLIMITED );
        w.setUseCompoundFile(false);
        w.addIndexesNoOptimize( d );
        if ( optimize )
          {
            w.optimize( );
          }
        w.commit( );
        w.close( );
      }
    catch ( IOException ioe )
      {
        System.err.println( "Error: " + args[0] + " " + ioe );

        if ( w != null ) w.close();
      }    
  }
}

@@ -957,16 +957,26 @@
 r %5B-o%7C-m
+%7C-b %3Csize%3E
 %5D %3Cdest%3E
@@ -1088,16 +1088,219 @@
 false;%0A%0A
+    // Default size of RAM buffer for merging indexes is max memory - 12MB;%0A    double bufsize = (Runtime.getRuntime().maxMemory() / 1024 / 1024) - 12;%0A    System.err.println( %22bufsize = %22 + bufsize );%0A%0A
     int 
@@ -1550,52 +1550,752 @@
 else
-%0A          %7B%0A            break ;%0A          %7D
+ if ( %22-b%22.equals( args%5Bi%5D ) )%0A          %7B%0A            if ( ++i %3E= args.length )%0A              %7B%0A                System.err.println( %22Missing ram buffer size, after -b%22 );%0A                System.exit(1);%0A              %7D%0A            try%0A              %7B%0A                bufsize = Double.parseDouble( args%5Bi%5D );%0A              %7D%0A            catch ( NumberFormatException nfe )%0A              %7B%0A                System.err.println( %22Invalid parameter, after -b: %22 + args%5Bi%5D );%0A                System.exit(1);%0A              %7D%0A          %7D%0A        else%0A          %7B%0A            break ;%0A          %7D%0A      %7D%0A%0A    if ( (args.length - i) %3C 2 )%0A      %7B%0A        System.err.println( %22IndexMerger %5B-o%7C-m%7C-b %3Csize%3E%5D %3Cdest%3E %3Csource%3E...%22 );%0A        System.exit( 1 );
 %0A   
@@ -2832,24 +2832,63 @@
 NLIMITED );%0A
+        w.setRAMBufferSizeMB(bufsize);%0A
         w.se
@@ -2936,18 +2936,8 @@
 exes
-NoOptimize
 ( d 
@@ -2998,17 +2998,16 @@
 ptimize(
- 
 );%0A     

Use most of available memory for buffering documents during merge. Add -b option to control that amount.