@@ -0,0 +1,2753 @@
+import pandas as pd%0Aimport numpy as np%0Aimport operator%0Aimport os %0Aimport re%0Aimport glob%0A%0Adata_dir = '/data2/gmrtarch/cycle20/'%0AVALID_LIST = 'parser/filter_healthy/healthy2_file.txt'%0Avalid_observations = open(VALID_LIST, 'r').read().split('%5Cn')%5B0:-1%5D%0Aall_observations = os.listdir(data_dir)%0Altahdr_error_dir = open('ltahdr_error.txt', 'w')%0A%0Adef INVALID_OBS():%0A    for DIR_NAME in all_observations:%0A        current_obslog = glob.glob(data_dir+DIR_NAME+'/'+'*.obslog')%0A        %22%22%22%0A        if current_obslog == %5B%5D:%0A            print DIR_NAME%0A            break%0A        %22%22%22%0A        #Extract substring that contains obslog relative path%0A        relative_path = re.findall(r'%5B/%5D%5B%5Cd%5D+%5B.%5Dobslog', current_obslog%5B0%5D)%5B0%5D%5B1:%5D %0A        #Invalid file (not fitting given constraints i.e. %3C 900 MHz and IF BW != 6,16,32)%0A        if relative_path not in valid_observations:%0A            print data_dir+DIR_NAME%0A        #Valid obslog file with no LTA file in the DIR%0A        %0A        if relative_path in valid_observations:%0A            if glob.glob(data_dir+DIR_NAME+'/'+'*.lta') == %5B%5D:%0A                print data_dir+DIR_NAME%0A%0Adef VALID_OBS():%0A    valid_obs = %5B%5D%0A    for DIR_NAME in all_observations:%0A            current_obslog = glob.glob(data_dir+DIR_NAME+'/'+'*.obslog')%0A            %0A            #Extract substring that contains obslog relative path%0A            relative_path = re.findall(r'%5B/%5D%5B%5Cd%5D+%5B.%5Dobslog', current_obslog%5B0%5D)%5B0%5D%5B1:%5D %0A                                    %0A            if relative_path in valid_observations:%0A                if glob.glob(data_dir+DIR_NAME+'/'+'*.lta') != %5B%5D:%0A                    valid_obs.append(data_dir+DIR_NAME)%0A    return valid_obs%0A%0Avalid_observations = VALID_OBS()%0A%0Afor DIR in valid_observations:%0A    os.chdir(DIR)%0A    all_lta_files = glob.glob('*.lta')%0A    for LTA_FILE in all_lta_files:%0A        try:%0A            print main(LTA_FILE)%0A        except:%0A            #Log the error directory to a file%0A            ltahdr_error_dir.write(DIR+ '/' + LTA_FILE)%0A            %0Adef extract( file_name ):%0A    with open(file_name) as f:%0A        for i,line in enumerate(f,1):%0A            if %22SCN%22 in line:%0A                return i%0A%0Adef main(lta_name):%0A    os.system('ltahdr -i'+ lta_name + '%3E lta_file.txt')%0A    dictionary = %7B%7D%0A    skipped_rows = extract('lta_file.txt')-1%0A%0A    header = pd.read_csv('lta_file.txt',skiprows=skipped_rows,delimiter=r%22%5Cs+%22)%0A    flux = list(set(header%5B%22OBJECT%22%5D))%0A    #print flux%0A%0A    header%5B'Nrecs'%5D = header%5B'Nrecs'%5D.astype(float)%0A%0A    for i in flux :%0A        temp = header.loc%5Bheader.OBJECT==i,'Nrecs'%5D.values%0A        temp = np.mean(temp)%0A        dictionary%5Bi%5D=temp%0A    #print dictionary%0A%0A    source = max(dictionary.iteritems(),key=operator.itemgetter(1))%5B0%5D%0A    os.system('rm lta_file.txt')%0A    return source%0A

Add file to test the validity script before integration