# -*- coding: utf-8 -*-
import sys
import os
import re
import yaml
import json
import time
import csv
import requests
import glob
import subprocess
import Queue
import threading

NUM_CONNECTION = 5
NUM_PER_REQUEST = 400

def http_post(queue, url, api_token, message_type):
    while True:
        try:
            params = queue.get()
            if params is None:
                break
            data = {
                "api_token": api_token,
                "logs": []
            }
            for param in params:
                data['logs'].append({
                    "type": message_type,
                    "attributes": param
                })
            r = requests.post(url, data=json.dumps(data))
            if r.status_code == requests.codes.ok:
                print "[SUCCESS] Sent record {0}.".format(len(params))
            else:
                print "[ERROR] Status: {0}, {1}".format(r.status_code, r.text)
            queue.task_done()
        except Exception as ex:
            print str(ex)

def read_csvfile(queue, csv_file, raspi_mac_addr, url, api_token, message_type):
    data_per_request = []
    data = []
    f = open(csv_file, 'r')
    reader = csv.reader(f)
    header = next(reader)
    table = {}
    for o in reader:
        if len(o) == 3:
            ts = o[0]
            src_mac = o[1]
            rssi = o[2]
            if ts and src_mac and rssi and raspi_mac_addr:
                ts = ts.split(".")[0]
                if table.has_key(ts + "." + src_mac):
                    # print "Skip duplicate data. ts={0} mac={1}".format(ts, src_mac)
                    continue
                table[ts + "." + src_mac] = True
                data_per_request.append({
                    "timestamp": ts,
                    "src_mac": src_mac,
                    "rssi": rssi,
                    "raspi_mac": raspi_mac_addr,
                })

            size = len(data_per_request)
            if size >= NUM_PER_REQUEST:
                queue.put(data_per_request)
                data_per_request = []
        # else:
        #     print "Skip invalid data."

    size = len(data_per_request)
    if size > 0:
        queue.put(data_per_request)
    f.close()

def main(target_dir, raspi_mac_addr, config_path):
    config = yaml.load(open(config_path, 'r'))
    url = config['url']
    api_token = config['api_token']
    message_type = config['message_type']

    request_queue = Queue.Queue(maxsize=1)

    for i in range(NUM_CONNECTION):
        threading.Thread(target=http_post, args=(request_queue, url, api_token, message_type)).start()

    # process pcap files
    while True:
        files = glob.glob(target_dir + "/packet_*.pcap")
        if len(files) <= 1:
            time.sleep(1.0)
            continue
        files.sort()
        files.pop()
        for f in files:
            csv_file = f + ".csv"
            subprocess.call("sudo tshark -r '{}' -T fields -E separator=',' -e frame.time_epoch -e wlan.sa -e radiotap.dbm_antsignal > '{}'".format(f, csv_file), shell=True)
            read_csvfile(request_queue, csv_file, raspi_mac_addr, url, api_token, message_type)
            os.remove(csv_file)
            os.remove(f)

    request_queue.join()


# Usage sudo post_data.py target_dir raspi_mac_addr config_path
if __name__ == '__main__':
    main(sys.argv[1], sys.argv[2], sys.argv[3])


@@ -655,16 +655,53 @@
      %7D)%0A
+            start_time = time.time()%0A
         
@@ -750,16 +750,54 @@
 (data))%0A
+            finish_time = time.time()%0A
         
@@ -887,16 +887,28 @@
 cord %7B0%7D
+ in %7B1%7D secs
 .%22.forma
@@ -920,16 +920,42 @@
 (params)
+, finish_time - start_time
 )%0A      
@@ -1013,16 +1013,28 @@
 %7B0%7D, %7B1%7D
+ in %7B2%7D secs
 %22.format
@@ -1055,16 +1055,42 @@
 , r.text
+, finish_time - start_time
 )%0A      
@@ -2125,16 +2125,56 @@
 EQUEST:%0A
+                push_time = time.time()%0A
         
@@ -2243,24 +2243,245 @@
 equest = %5B%5D%0A
+                pushed_time = time.time()%0A                wait_duration = pushed_time - push_time%0A                if wait_duration %3E 10:%0A                    print(%22Request queue push wait %7B%7D secs.%22.format(wait_duration))%0A
         # el
@@ -2576,16 +2576,48 @@
 ze %3E 0:%0A
+        push_time = time.time()%0A
         
@@ -2640,24 +2640,179 @@
 er_request)%0A
+        wait_duration = pushed_time - push_time%0A        if wait_duration %3E 10:%0A            print(%22Request queue push wait %7B%7D secs.%22.format(wait_duration))%0A
     f.close(
@@ -3473,16 +3473,74 @@
  %22.csv%22%0A
+            print(%22Dumping %7B%7D -%3E %7B%7D%22.format(f, csv_file))%0A
         

Add some log messages.