#!/usr/bin/env python
import socket, sys

import logging
logging.getLogger("scapy.runtime").setLevel(logging.ERROR)

from scapy.all import *
from subprocess import call	


#CMD TO DUMP INCOMING/OUTGOING TCP DATA AT THE CONTROLLER - send to socket directly
proc = subprocess.Popen('sudo tshark -i wlan0 -T fields -e frame.number -e frame.interface_id -e frame.encap_type -e frame.offset_shift -e frame.time_epoch -e frame.time_delta -e frame.time_delta_displayed -e frame.time_relative -e frame.len -e frame.cap_len -e frame.marked -e frame.ignored -e frame.protocols -e ip.version -e ip.hdr_len -e ip.dsfield -e ip.dsfield.dscp -e ip.dsfield.ecn -e ip.len -e ip.id -e ip.flags -e ip.flags.rb -e ip.flags.df -e ip.flags.mf -e ip.frag_offset -e ip.ttl -e ip.proto -e tcp.srcport -e tcp.dstport -e tcp.stream -e tcp.hdr_len -e tcp.flags -e tcp.flags.res -e tcp.flags.ns -e tcp.flags.cwr -e tcp.flags.ecn -e tcp.flags.urg -e tcp.flags.ack -e tcp.flags.push -e tcp.flags.reset -e tcp.flags.syn -e tcp.flags.fin -e tcp.window_size_value -e tcp.window_size -e tcp.window_size_scalefactor -e tcp.option_len -e tcp.options.timestamp.tsval -e tcp.options.timestamp.tsecr -e tcp.analysis.bytes_in_flight -e eth.lg -e eth.ig -e eth.type -e arp.hw.type -e arp.proto.type -e arp.hw.size -e arp.proto.size -e arp.opcode -E header=y -E separator=,',
			shell=True,
			stdin=subprocess.PIPE,
      stdout=subprocess.PIPE,		# Write to a buffer instead of a file/pipe
      stderr=subprocess.PIPE)


# Pre-process data
	''' by replacing NULLs with 0
	encoding categorical attributes'''


# Check/modify the following code to send the data from the pre-processed buffer to EC2

''' 
suppose array data[] contains the prepared data from the previous steps, 
now send each item in a for loop as a socket client.
actually the previous packet sampler is a socket server running in a 
while (True) loop to keep collecting data, and the Spark
part acts as a socket client.
'''

# hard-coded IP and port
# need to replace it with EC2 instance parameter
TCP_IP = '127.0.0.1'
TCP_PORT = 3000
BUFFER_SIZE = 1024

s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((TCP_IP, TCP_PORT))
for item in data
  s.send(item)
s.close()


@@ -38,246 +38,746 @@
 sys%0A
-%0A
 import 
-logging%0Alogging.getLogger(%22scapy.runtime%22).setLevel(logging.ERROR)%0A%0Afrom scapy.all import *%0Afrom subprocess import call%09%0A%0A%0A#CMD TO DUMP INCOMING/OUTGOING TCP DATA AT THE CONTROLLER - send to socket directly%0Aproc = subprocess.Popen
+subprocess, shlex%0A%0A# TODO: parsing one line of data from tshark%0A# need to:%0A# 1) encode hex data using unpack:%0A# http://stackoverflow.com/questions/3531723/unpack-from-hex-to-double-in-python%0A# 2) encode NULL to 0;%0A# 3) encode categorial data to numbers%0Adef parseSample(line):%0A  return line%0A%0A# connect as a TCP *client*%0A# should ideally be a server, because Spark should be the client%0A# but making this part a client is easier than letting EC2 connect%0A# to a machine inside university network%0A  %0A# hard-coded IP and port%0A# need to replace it with EC2 instance parameter%0ATCP_IP = '127.0.0.1'%0ATCP_PORT = 3000%0ABUFFER_SIZE = 1024%0Asoc = socket.socket(socket.AF_INET, socket.SOCK_STREAM)%0Asoc.connect((TCP_IP, TCP_PORT))%0A%0Acmd = shlex.split
 ('su
@@ -1829,28 +1829,53 @@
 r=,'
-,%0A%09%09%09shell=True,%0A%09%09%09
+)%0A%0Aproc = subprocess.Popen(cmd, shell=False, 
 stdi
@@ -1892,22 +1892,16 @@
 ss.PIPE,
-%0A     
  stdout=
@@ -1920,791 +1920,316 @@
 IPE,
-%09%09# Write to a buffer instead of a file/pipe%0A      stderr=subprocess.PIPE)%0A%0A%0A# Pre-process data%0A%09''' by replacing NULLs with 0%0A%09encoding categorical attributes'''%0A%0A%0A# Check/modify the following code to send the data from the pre-processed buffer to EC2%0A%0A''' %0Asuppose array data%5B%5D contains the prepared data from the previous steps, %0Anow send each item in a for loop as a socket client.%0Aactually the previous packet sampler is a socket server running in a %0Awhile (True) loop to keep collecting data, and the Spark%0Apart acts as a socket client.%0A'''%0A%0A# hard-coded IP and port%0A# need to replace it with EC2 instance parameter%0ATCP_IP = '127.0.0.1'%0ATCP_PORT = 3000%0ABUFFER_SIZE = 1024%0A%0As = socket.socket(socket.AF_INET, socket.SOCK_STREAM)%0As.connect((TCP_IP, TCP_PORT))%0Afor item in data
+ stderr=subprocess.PIPE)%0A# proc = subprocess.Popen(%5B'ping', 'google.com'%5D, shell=False, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)%0A%0A# skip the first line, it is not data%0Aproc.stdout.readline()%0A%0A# keep reading and sending%0Awhile 1:%0A  item = parseSample(proc.stdout.readline())
 %0A  s
+oc
 .sen
@@ -2236,17 +2236,20 @@
 d(item)%0A
-s
+%0Asoc
 .close()

put tshark data sampling in a while 1 loop; raw data not parsed yet