# Copyright (c) 2012 G. Hemingway
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import re
import hashlib

class LogProcessor(object):
    """
    Handle the processing of a single type of log
    """
    def __init__(self, logger, config):
        try:
            self.logger = logger
            self.label = config['label']
            self.container = config['container']
            self.hostname = config['hostname']
            self.format = config['format']
            self.directory = config['directory']
            self.file_name = config['file_name']
            self.compress = config['compress']
            self.remove = config['remove']

        except Exception as e:
            logger.fatal('Failed to initialize LogProcessor')
            exit(-1)


    def _filter_files(self):
        """
        Filter files based on regex pattern
        :param all_files: list of strs, relpath of the filenames under self.directory
        :param pattern: regex pattern to match against filenames
        :returns : dict mapping full path of file to match group dict
        """
        all_files = []
        for path, dirs, files in os.walk(self.directory):
            all_files.extend(os.path.join(path, f) for f in files)
        all_files = [os.path.relpath(f, start=self.directory) for f in all_files]
        pattern = '''^%s-(?P<year>[0-9]{4})
                         (?P<month>[0-1][0-9])
                         (?P<day>[0-3][0-9])
                         (?P<hour>[0-2][0-9]).*$''' % self.file_name
        filename2match = {}
        for filename in all_files:
            match = re.match(pattern, filename, re.VERBOSE)
            if match:
                full_path = os.path.join(self.directory, filename)
                filename2match[full_path] = match.groupdict()

        return filename2match


    def _determine_key(self, year, month, day, hour):
        """
        Build the object key name
        """
        values = []
        for pos,format in enumerate(self.format):
            if format == 'host':
                values.append(self.hostname)
            elif format == 'date':
                values += [year, month, day, hour]
            elif format == 'label':
                values.append(self.label)
        return '/'.join(values)


    def process(self, conn):
        """
        Process the uploading of one log file
        """
        file_list = self._filter_files()
        for filename, match in file_list.items():
            # Determine the key name
            key_name = self._determine_key(**match)
            if os.path.getsize(filename) != 0:
                self.logger.debug("Processing log: %s" % filename)
                hash = hashlib.md5()
                # Do we compress
                already_compressed = True if filename.endswith('.gz') else False
                if self.compress and not already_compressed:
                    source_file = gzip.open(filename, 'rb')
                    for line in source_file:
                        # filter out bad lines here?
                        hash.update(line)
                else:
                    source_file = open(filename, 'rb')
                hash = hash.hexdigest()
                # Upload the file
                self.logger.debug('Uploading %s.' % filename)
                headers = {'x-object-meta-original-name': filename, 'x-hash': hash }
                conn.put_object(self.container, key_name, source_file,
                    content_type = 'text/directory',
                    headers = headers)
                # Determine if we need to remove the file
                if self.remove:
                    self.logger.info("Deleting local copy of %s" % filename)


@@ -602,23 +602,36 @@
 %0Aimport 
-hashlib
+gzip%0Aimport tempfile
 %0A%0Aclass 
@@ -1172,17 +1172,16 @@
 emove'%5D%0A
-%0A
         
@@ -3000,45 +3000,8 @@
 ():%0A
-            # Determine the key name%0A
     
@@ -3123,21 +3123,20 @@
 .logger.
-debug
+info
 (%22Proces
@@ -3181,203 +3181,185 @@
     
-hash = hashlib.md5()%0A                # Do we compress%0A                already_compressed = True if filename.endswith('.gz') else False%0A                if self.compress and not already_compressed:
+tmpfile = ''%0A                if self.compress and not filename.endswith('.gz'):%0A                    key_name += '.gz'%0A                    (fd, tmpfile) = tempfile.mkstemp('.gz')
 %0A   
@@ -3393,13 +3393,8 @@
 e = 
-gzip.
 open
@@ -3434,149 +3434,176 @@
     
-for line in source_file:%0A                        # filter out bad lines here?%0A                        hash.update(line)%0A                else:
+gzip_file = gzip.open(tmpfile, 'wb')%0A                    gzip_file.writelines(source_file)%0A                    gzip_file.close()%0A                    source_file.close()
 %0A   
@@ -3630,39 +3630,38 @@
 rce_file = open(
+tmp
 fil
-enam
 e, 'rb')%0A       
@@ -3673,65 +3673,68 @@
     
-hash = hash.hexdigest()%0A                # Upload the file
+else:%0A                    source_file = open(filename, 'rb')
 %0A   
@@ -3796,93 +3796,8 @@
 me)%0A
-                headers = %7B'x-object-meta-original-name': filename, 'x-hash': hash %7D%0A
     
@@ -3946,23 +3946,18 @@
 aders = 
-headers
+%7B%7D
 )%0A      
@@ -3970,49 +3970,27 @@
     
-# Determine if we need to remove the file
+source_file.close()
 %0A   
@@ -4050,20 +4050,21 @@
 .logger.
-info
+debug
 (%22Deleti
@@ -4095,10 +4095,288 @@
 ilename)
+%0A                    try:%0A                        os.remove(filename)%0A                    except Exception as e:%0A                        self.logger.error('Exception while trying to delete %25s.' %25 filename)%0A                if tmpfile != '':%0A                    os.remove(tmpfile)
 %0A%0A

Fix compress and remove flag behavior