"""
Get specified number of random tweets from CSV with over 100K entries

"""

#-------------------------------------------#
# Import dependencies                       #
#-------------------------------------------#

import time
import re
import string
import numpy as np
import pandas as pd
import random as rnd
import multiprocessing as mp
import csv


from datetime import datetime, timedelta


#-------------------------------------------#
# Global variables                          #
#-------------------------------------------#

CHUNKSIZE = 10000                                                                          # processing 100,000 rows at a time
WORKER_PROCESSES = 4
SAMPLE_SIZE = 62200
USERID = '2980186012'
INPUT = '../datasets/top_users_csv/{}.csv'.format(USERID)
OUTPUT = '../datasets/top_users_csv/{}_{}.csv'.format(SAMPLE_SIZE, USERID)

#-------------------------------------------#
# Set random seed                           #
#-------------------------------------------#
rnd.seed(8)

#-------------------------------------------#
# Columns of CSV file                       #
#-------------------------------------------#

ibm_cols = [
    'content',
    'id',
    'created',
    'contenttype',
    'language',
    'parentid',
    'reply',
    'forward'
]

#-------------------------------------------#
# D-types                                   #
#-------------------------------------------#

ibm_d_types = {
    'content':                  'str',                                                      # tweet_text
    'id':                       'str',                                                      # tweetid
    'created':                  'int',                                                      # tweet_time (needs to be converted to UNIX)
    'contenttype':              'str',                                                      # set default value as 'text/plain'
    'language':                 'str',                                                      # set default value as 'en' since we check this in cleaning step
    'parentid':                 'str',                                                      # set default value as userid
    'reply':                    'str',                                                     # check if in_reply_to_tweetid/userid is filled or not, return bool
    'forward':                  'str',                                                     # check if is_retweet is True or False, return bool

}

#-------------------------------------------#
# Function                                  #
#-------------------------------------------#

def process_frame(df):
    # print(df.tweet_text)
    return len(df)                                                                          # process data frame

#-------------------------------------------#
# Main                                      #
#-------------------------------------------#

def main():
    start_time = time.time()                                                                # program start time
    print("Processing file: ",INPUT)

    reader = pd.read_csv(
                         INPUT,
                         header=0,
                         dtype=ibm_d_types,
                         chunksize=CHUNKSIZE,
                         keep_default_na=False,                                  # do not parse rows with default NaN values
                        )
    pool = mp.Pool(4)                                                               # use 4 processes

    funclist = []
    reader_count = 1
    for df in reader:
        print("Processing %d df of reader..."%(reader_count))
        f = pool.apply_async(process_frame,[df])
        funclist.append(f)
        reader_count += 1

    n = 0
    for f in funclist:
        n += f.get(timeout=10)                                                      # timeout in 10 seconds

    print("There are ",n," lines in ",INPUT)
    skip = sorted(rnd.sample(range(1,n+1),n-SAMPLE_SIZE))                           # the 0-indexed header will not be included in the skip list
    df = pd.read_csv(
                    INPUT, 
                    skiprows=skip
                    )
    df.to_csv(
                OUTPUT,
                columns=ibm_cols, 
                encoding='utf-8', 
                index=False, 
                header=True                                                           # include header for first chunk
            )

    runtime = "Execution: %s seconds" % timedelta(seconds=time.time() - start_time)         # calculate runtime
    print(runtime)

#-------------------------------------------#
# Call Main()                               #
#-------------------------------------------#

if __name__ == "__main__":
    main()