#!usr/bin/env python

#===============================================================================
# Import modules
#===============================================================================
# Standard Library
import os
import sys
import datetime
import time
import collections
from xml.etree import cElementTree as ET

# Third party modules

# Application modules
import toolbox.log as log
import toolbox.check_process as check_process
import toolbox.xml_tools as xml_tools


#===============================================================================
# MAIN
#===============================================================================
def main():

    '''
    Grabs data from all yearly xml files and consolidates the data for web heat
    calendars.
    '''
   
    script_name = os.path.basename(sys.argv[0])
    folder_loc  = os.path.dirname(os.path.realpath(sys.argv[0]))
    folder_loc  = folder_loc.replace('garden-web-server/scripts', 'weather')


    #---------------------------------------------------------------------------
    # Set up logger
    #---------------------------------------------------------------------------
    logger = log.setup('root', '{folder}/logs/{script}.log'.format(
                                                    folder= folder_loc,
                                                    script= script_name[:-3]))

    logger.info('')
    logger.info('--- Script {script} Started ---'.format(script= script_name)) 


    #---------------------------------------------------------------------------
    # CHECK SCRIPT IS NOT ALREADY RUNNING
    #--------------------------------------------------------------------------- 
    if check_process.is_running(script_name):
        logger.info('Script already runnning. Exiting...')
        logger.info(other_script_found)
        sys.exit()  


    try:
        #-------------------------------------------------------------------
        # Get data from yearly files
        #-------------------------------------------------------------------      
        sensors = []
        xml_data = {}

        years = [str(year) for year in range(2016, datetime.datetime.now().year+1)]

        for year_ref in years:

            xml_file = 'wd_all_{d}.xml'.format(d= year_ref)

            logger.info('Getting data from {f}'.format(f= xml_file)) 
            tree = ET.parse('{folder}/data/{xml_file}'.format(
                                                        folder= folder_loc,
                                                        xml_file= xml_file))
            logger.info('Sorting data')
            
            root = tree.getroot()
            xmldict = xml_tools.XmlDictConfig(root)

            for sensor in root.iter('entry'):
                if sensor.text not in sensors:
                    sensors.append(sensor.text)
                    xml_data[sensor.text] = {}

                if year_ref not in xml_data[sensor.text].keys():   
                    xml_data[sensor.text][year_ref] = {}


            # Sort data under sensor and then time
            for child in root[1]:
                for grandchild in child:
                    if grandchild.tag == 't':
                        time = str(int(grandchild.text) / 1000)
                        i = 0
                    elif grandchild.tag == 'v':
                        xml_data[sensors[i]][year_ref][time] = grandchild.text
                        i += 1


        #-------------------------------------------------------------------
        # Remove min and max data sets
        #-------------------------------------------------------------------
        for sensor in xml_data.keys():
            if '_min' in sensor or '_max' in sensor:
                del xml_data[sensor]


        #-------------------------------------------------------------------
        # Write data to xml file
        #-------------------------------------------------------------------

        output_file = 'wd_all_years.xml'
        logger.info('Writting data to {f}'.format(f=output_file))
        # create new xml
        root = ET.Element('root')
        tree = ET.ElementTree(root)
        data = ET.SubElement(root, 'data')

        # populate xml
        for op_xml in xml_data.keys():

            # Create a subelement for each sensor 
            sensor = ET.SubElement(data, 'sensor', name= op_xml) 

            for year_ref in years:
                year = ET.SubElement(sensor, 'year', year= year_ref)

                # Populate time and value in each sensor          
                for time, value in xml_data[op_xml][year_ref].iteritems():
                    ET.SubElement(year, 'entry', time=time).text = value
        

        logger.info('{folder}/data/{xml_file}'.format(folder= folder_loc,
                                                    xml_file= output_file))
        tree.write('{folder}/data/{xml_file}'.format(folder= folder_loc,
                                                    xml_file= output_file))


    except Exception, e:
        logger.error('ERROR: ({error_v}).'.format(error_v=e), exc_info=True)
        sys.exit()

    finally:
        logger.info('--- Script {script} Finished ---'.format(script= script_name)) 

 


#===============================================================================
# Boiler plate
#===============================================================================
if __name__=='__main__':
    main()

@@ -281,16 +281,28 @@
 ections%0A
+import copy%0A
 from xml
@@ -2576,61 +2576,8 @@
 e))%0A
-            logger.info('Sorting data')%0A            %0A
     
@@ -2606,16 +2606,17 @@
 troot()%0A
+%0A
         
@@ -2623,307 +2623,114 @@
     
-xmldict = xml_tools.XmlDictConfig(root)%0A%0A            for sensor in root.iter('entry'):%0A                if sensor.text not in sensors:%0A                    sensors.append(sensor.text)%0A                    xml_data%5Bsensor.text%5D = %7B%7D%0A%0A                if year_ref not in xml_data%5Bsensor.text%5D.keys():   %0A 
+if not xml_data:%0A                logger.info('First copy')%0A                xml_data = copy.deepcopy(tree)%0A
     
@@ -2745,1766 +2745,523 @@
     
-   
 xml_
-data%5Bsensor.text%5D%5Byear_ref%5D = %7B%7D%0A%0A%0A            # Sort data under sensor and then time%0A            for child in root%5B1%5D:%0A                for grandchild in child:%0A                    if grandchild.tag == 't':%0A                        time = str(int(grandchild.text) / 1000)%0A                        i = 0%0A                    elif grandchild.tag == 'v':%0A                        xml_data%5Bsensors%5Bi%5D%5D%5Byear_ref%5D%5Btime%5D = grandchild.text%0A                        i += 1%0A%0A%0A        #-------------------------------------------------------------------%0A        # Remove min and max data sets%0A        #-------------------------------------------------------------------%0A        for sensor in xml_data.keys():%0A            if '_min' in sensor or '_max' in sensor:%0A                del xml_data%5Bsensor%5D%0A%0A%0A        #-------------------------------------------------------------------%0A        # Write data to xml file%0A        #-------------------------------------------------------------------%0A%0A        output_file = 'wd_all_years.xml'%0A        logger.info('Writting data to %7Bf%7D'.format(f=output_file))%0A        # create new xml%0A        root = ET.Element('root')%0A        tree = ET.ElementTree(root)%0A        data = ET.SubElement(root, 'data')%0A%0A        # populate xml%0A        for op_xml in xml_data.keys():%0A%0A            # Create a subelement for each sensor %0A            sensor = ET.SubElement(data, 'sensor', name= op_xml) %0A%0A            for year_ref in years:%0A                year = ET.SubElement(sensor, 'year', year= year_ref)%0A%0A                # Populate time and value in each sensor          %0A                for time, value in xml_data%5Bop_xml%5D%5Byear_ref%5D.iteritems():%0A                    ET.SubElement(year, 'entry', time=time).text = value%0A        %0A%0A        logger.info('
+root = xml_data.getroot()%0A                continue%0A%0A            data = root%5B1%5D%0A%0A            # append year's data to file%0A            for row in root%5B1%5D.findall('row'):%0A                print row%5B0%5D.text%0A                xml_root%5B1%5D.append(row)%0A%0A%0A        #-------------------------------------------------------------------%0A        # Write data to xml file%0A        #-------------------------------------------------------------------%0A%0A        output_file = 'wd_all_years.xml'%0A        logger.info('Writting data to 
 %7Bfol
@@ -3393,20 +3393,24 @@
         
-tree
+xml_data
 .write('

Reformat output to match existing files