@@ -0,0 +1,1964 @@
+#!/usr/bin/env python%0Afrom collections import defaultdict%0Aimport csv%0Aimport datetime%0Aimport glob%0Aimport os%0A%0Aimport numpy as np%0Afrom sklearn import metrics%0Afrom sklearn.cluster import KMeans%0A%0A%0Adef extract_feature_vectors(data_dir):%0A    node_dirs = os.listdir(data_dir)%0A%0A    # Find all the log types%0A    all_log_types = set()%0A    for node_dir in node_dirs:%0A        log_files = glob.glob(os.path.join(data_dir, node_dir, '*.csv'))%0A%0A        all_log_types.update(%5B%0A            os.path.splitext(os.path.basename(log_file))%5B0%5D%0A            for log_file in log_files%5D)%0A    all_log_types = frozenset(all_log_types)%0A%0A    # Extract feature vectors%0A    fvs_by_node = defaultdict(lambda: np.zeros(len(all_log_types), dtype=np.int))%0A    for node_dir in node_dirs:%0A        for i, log_type in enumerate(all_log_types):%0A            with open(os.path.join(data_dir, node_dir, '%25s.csv' %25 log_type), 'rb') as f:%0A                reader = csv.DictReader(f)%0A                rows = list(reader)  # A header is not included.%0A%0A                # Get the date column%0A                dates = %5B%5D%0A                for row in rows:%0A                    try:%0A                        date = datetime.datetime.strptime(row%5B'date'%5D,%0A                                '%25Y-%25m-%25d %25H:%25M:%25S.%25f')%0A                    except ValueError:%0A                        date = datetime.datetime.strptime(row%5B'date'%5D,%0A                                '%25Y-%25m-%25d %25H:%25M:%25S')%0A                    dates.append(date)%0A%0A                # Might need to filter by timestamps%0A                fvs_by_node%5Bnode_dir%5D%5Bi%5D = len(rows)%0A    return fvs_by_node, all_log_types%0A%0A%0Adef cluster(fvs, n_clusters):%0A    fvs = np.asarray(fvs)%0A    kmeans = KMeans(init='k-means++', n_clusters=n_clusters)%0A    return kmeans.fit_predict(fvs)%0A%0A%0Aif __name__ == '__main__':%0A    fvs_by_node, all_log_types = extract_feature_vectors('../parsedData/')%0A    fvs_np = np.array(fvs_by_node.values())%0A%0A    print 3, cluster(fvs_np, 3)%0A    print 2, cluster(fvs_np, 2)%0A

Add clusterer directory