#!/usr/bin/python

'''
Program to process a whole directory full of compressed source code archives
to create a knowledgebase. Needs a file LIST in the directory it is passed as
a parameter.

package version filename

seperated by whitespace.

Compression is determined using magic
'''

import sys, os, magic
import tempfile, bz2, tarfile, gzip
from optparse import OptionParser
import extractprogramstrings

tarmagic = ['POSIX tar archive (GNU)'
           , 'tar archive'
           ]

def unpack(dir, filename):
        ms = magic.open(magic.MAGIC_NONE)
        ms.load()
        filemagic = ms.file("%s/%s" % (dir, filename))

        ## just assume if it is bz2 or gzip that we are looking at tar files with compression

        if 'bzip2 compressed data' in filemagic:
	        tar = tarfile.open("%s/%s" % (dir, filename), 'r:bz2')
       		tmpdir = tempfile.mkdtemp()
       		tar.extractall(path=tmpdir)
        	tar.close()
		return tmpdir
        elif 'gzip compressed data' in filemagic:
	        tar = tarfile.open("%s/%s" % (dir, filename), 'r:gz')
       		tmpdir = tempfile.mkdtemp()
       		tar.extractall(path=tmpdir)
        	tar.close()
		return tmpdir

def main(argv):
        parser = OptionParser()
        parser.add_option("-d", "--database", action="store", dest="db", help="path to Lucene database)", metavar="DIR")
        parser.add_option("-f", "--files", action="store", dest="filedir", help="path to directory containing files to unpack)", metavar="DIR")
        parser.add_option("-z", "--cleanup", action="store_true", dest="cleanup", help="cleanup after unpacking? (default: true)")
        (options, args) = parser.parse_args()


        ## TODO: do all kinds of checks here
        filelist = open(options.filedir + "/LIST").readlines()
        for unpackfile in filelist:
                (package, version, filename) = unpackfile.strip().split()
                temporarydir = unpack(options.filedir, filename)
		extractprogramstrings.main(["-i", "/tmp/lucene", "-d", temporarydir, "-p", package, "-v", version])

if __name__ == "__main__":
        main(sys.argv)

@@ -1481,32 +1481,185 @@
 =%22DIR%22)%0A        
+parser.add_option(%22-v%22, %22--verify%22, action=%22store_true%22, dest=%22verify%22, help=%22verify files, don't process (default: false)%22)%0A%09# implement later%0A        #
 parser.add_optio
@@ -1765,16 +1765,16 @@
 true)%22)%0A
-
         
@@ -2031,22 +2031,166 @@
 split()%0A
-      
+%09%09if options.verify:%0A%09%09%09try:%0A%09%09%09%09os.stat(%22%25s/%25s%22 %25 (options.filedir, filename))%0A%09%09%09except:%0A%09%09%09%09print %3E%3Esys.stderr, %22Can't find %25s%22 %25 filename%0A%09%09else:%0A
         
@@ -2191,16 +2191,23 @@
         
+      %09
 temporar
@@ -2247,16 +2247,17 @@
 lename)%0A
+%09
 %09%09extrac

add verify option to make sure the files specified in the batch list actually exist