/*
 * The Gemma project
 * 
 * Copyright (c) 2008 University of British Columbia
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */
package ubic.gemma.loader.util.fetcher;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.SocketException;
import java.net.UnknownHostException;
import java.util.Collection;
import java.util.HashSet;
import java.util.concurrent.Callable;
import java.util.concurrent.CancellationException;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.FutureTask;

import org.apache.commons.net.ftp.FTP;
import org.apache.commons.net.ftp.FTPClient;

import ubic.basecode.util.NetUtils;
import ubic.gemma.model.common.description.LocalFile;
import ubic.gemma.util.NetDatasourceUtil;

/**
 * Download files by FTP.
 * 
 * @author paul
 * @version $Id$
 */
public abstract class FtpFetcher extends AbstractFetcher {

    protected FTPClient ftpClient;

    protected NetDatasourceUtil netDataSourceUtil;

    /**
     * 
     */
    public FtpFetcher() {
        super();
        setNetDataSourceUtil();
    }

    /**
     * @param outputFileName
     * @param seekFile
     * @return
     */
    protected FutureTask<Boolean> defineTask( final String outputFileName, final String seekFile ) {
        FutureTask<Boolean> future = new FutureTask<Boolean>( new Callable<Boolean>() {
            @SuppressWarnings("synthetic-access")
            public Boolean call() throws FileNotFoundException, IOException {
                log.info( "Fetching " + seekFile + " to " + outputFileName );
                boolean status = NetUtils.ftpDownloadFile( ftpClient, seekFile, outputFileName, force );
                ftpClient.disconnect();
                return new Boolean( status );
            }
        } );
        return future;
    }

    public Collection<LocalFile> fetch( String identifier ) {

        String seekFile = formRemoteFilePath( identifier );

        return fetch( identifier, seekFile );
    }

    /**
     * @param identifier
     * @param seekFile
     * @return
     */
    protected Collection<LocalFile> fetch( String identifier, String seekFile ) {
        File existingFile = null;
        try {
            File newDir = mkdir( identifier );
            String outputFileName = formLocalFilePath( identifier, newDir );

            existingFile = new File( outputFileName );
            if ( existingFile.canRead() ) {
                log.info( outputFileName + " already exists: checking size match." );
            }

            if ( ftpClient == null || !ftpClient.isConnected() ) {
                ftpClient = this.getNetDataSourceUtil().connect( FTP.BINARY_FILE_TYPE );
                assert ftpClient != null; // otherwise should have gotten an exception from connect()
            }

            long expectedSize = getExpectedSize( seekFile );

            FutureTask<Boolean> future = this.defineTask( outputFileName, seekFile );
            Collection<LocalFile> result = this.doTask( future, expectedSize, seekFile, outputFileName );
            return result;
        } catch ( UnknownHostException e ) {
            if ( force || !allowUseExisting || existingFile == null ) throw new RuntimeException( e );

            log.warn( "Could not connect to " + this.getNetDataSourceUtil().getHost() + " to check size of " + seekFile
                    + ", using existing file" );
            Collection<LocalFile> fallback = getExistingFile( existingFile, seekFile );
            return fallback;
        } catch ( IOException e ) {
            
            if ( force || !allowUseExisting || existingFile == null ) throw new RuntimeException( "Couldn't fetch " + seekFile
                    + ", make sure the file exists on the remote server.", e );

            log.warn("Couldn't fetch " + seekFile + ", make sure the file exists on the remote server.," + e
                    + ", using existing file" );
            Collection<LocalFile> fallback = getExistingFile( existingFile, seekFile );
            return fallback;
            
        }
    }

    /**
     * @param seekFile
     * @return
     * @throws IOException
     * @throws SocketException
     */
    protected long getExpectedSize( final String seekFile ) throws IOException, SocketException {
        return NetUtils.ftpFileSize( ftpClient, seekFile );
    }

    /**
     * @param future
     * @param expectedSize
     * @param outputFileName
     * @param seekFileName
     * @return
     */
    protected Collection<LocalFile> doTask( FutureTask<Boolean> future, long expectedSize, String seekFileName,
            String outputFileName ) {

        ExecutorService executor = Executors.newSingleThreadExecutor();
        executor.execute( future );
        executor.shutdown();

        try {

            File outputFile = new File( outputFileName );
            boolean ok = waitForDownload( future, expectedSize, outputFile );

            if ( !ok ) {
                // cancelled, probably.
                return null;
            } else if ( future.get().booleanValue() ) {
                if ( log.isInfoEnabled() ) log.info( "Done: local file is " + outputFile );
                LocalFile file = fetchedFile( seekFileName, outputFile.getAbsolutePath() );
                Collection<LocalFile> result = new HashSet<LocalFile>();
                result.add( file );
                return result;
            }
        } catch ( ExecutionException e ) {
            throw new RuntimeException( "Couldn't fetch " + seekFileName, e );
        } catch ( InterruptedException e ) {
            log.warn( "Interrupted: Couldn't fetch " + seekFileName, e );
            return null;
        } catch ( CancellationException e ) {
            log.info( "Cancelled" );
            return null;
        }
        throw new RuntimeException( "Couldn't fetch file for " + seekFileName );
    }

    /**
     * @param netDataSourceUtil the netDataSourceUtil to set
     */
    public abstract void setNetDataSourceUtil();

    /**
     * @return the netDataSourceUtil
     */
    public NetDatasourceUtil getNetDataSourceUtil() {
        return this.netDataSourceUtil;
    }
}

@@ -4179,79 +4179,8 @@
     
-%0A            if ( force %7C%7C !allowUseExisting %7C%7C existingFile == null ) 
 thro
@@ -4315,297 +4315,8 @@
 e );
-%0A%0A            log.warn(%22Couldn't fetch %22 + seekFile + %22, make sure the file exists on the remote server.,%22 + e%0A                    + %22, using existing file%22 );%0A            Collection%3CLocalFile%3E fallback = getExistingFile( existingFile, seekFile );%0A            return fallback;%0A            
 %0A   
@@ -5247,16 +5247,82 @@
 obably.%0A
+                log.info( %22Download failed, was it cancelled?%22 );%0A
         

Revert "fallback" to use local file if remote file cannot be fetched. This is just too dangerous, it causes half-downloaded files to be used (just got bit by this).