from functools import partial

import numpy as np
import pandas as pd

from pyqtgraph.Qt import QtCore

from graphysio.types import PlotData

class Reader(QtCore.QRunnable):
    def __init__(self, csvrequest, sigdata, sigerror):
        super().__init__()
        self.csvrequest = csvrequest
        self.sigdata = sigdata
        self.sigerror = sigerror

    def run(self) -> None:
        try:
            data = self.getdata()
        except Exception as e:
            #raise e
            self.sigerror.emit(e)
        else:
            self.sigdata.emit(data)

    def getdata(self) -> PlotData:
        data = pd.read_csv(self.csvrequest.filepath,
                           sep       = self.csvrequest.seperator,
                           usecols   = self.csvrequest.fields,
                           decimal   = self.csvrequest.decimal,
                           skiprows  = self.csvrequest.droplines,
                           encoding  = self.csvrequest.encoding,
                           index_col = False,
                           engine    = 'c')

        pdtonum = partial(pd.to_numeric, errors='coerce')
        dtformat = self.csvrequest.datetime_format
        if self.csvrequest.generatex:
            data.index = (1e9 * data.index / self.csvrequest.samplerate).astype(np.int64)
        elif dtformat == '<infer>':
            timestamp = data[self.csvrequest.dtfield].dropna()
            timestamp = pd.to_datetime(timestamp, infer_datetime_format=True)
        else:
            timestamp = data[self.csvrequest.dtfield]
            if dtformat == '<seconds>':
                timestamp = pdtonum(timestamp).dropna()
                timestamp = pd.to_datetime(timestamp * 1e9, unit = 'ns')
            elif dtformat == '<milliseconds>':
                timestamp = pdtonum(timestamp).dropna()
                timestamp = pd.to_datetime(timestamp * 1e6, unit = 'ns')
            elif dtformat == '<microseconds>':
                timestamp = pdtonum(timestamp).dropna()
                timestamp = pd.to_datetime(timestamp * 1e3, unit = 'ns')
            elif dtformat == '<nanoseconds>':
                timestamp = pdtonum(timestamp).dropna()
                timestamp = pd.to_datetime(timestamp, unit = 'ns')
            else:
                timestamp = pd.to_datetime(timestamp, format = dtformat)
            timestamp = pd.Index(timestamp).tz_localize(self.csvrequest.timezone).tz_convert('UTC')
            timestamp = timestamp.astype(np.int64)
            data = data.set_index([timestamp])
            data = data.drop(columns=self.csvrequest.dtfield)

        # Coerce all columns to numeric and remove empty columns
        data = data.apply(pdtonum).dropna(axis='columns', how='all')
        data = data.dropna(axis='rows', how='all')
        data = data.sort_index()

        plotdata = PlotData(data = data, filepath = self.csvrequest.filepath)
        return plotdata

@@ -1314,190 +1314,665 @@
     
-elif dtform
+    # Make all data numeric and remove empty rows%0A            d
 at
+a
  =
-= '%3Cinfer%3E':%0A            timestamp = data%5Bself.csvrequest.dtfield%5D.dropna()%0A            timestamp = pd.to_datetime(timestamp, infer_datetime_format=True)%0A        else:
+ data.apply(pdtonum)%0A            data = data.dropna(axis='rows', how='all')%0A        else:%0A            datacolumns = data.columns.drop(self.csvrequest.dtfield)%0A            # Make all data numeric or drop if impossible%0A            data%5Bdatacolumns%5D = data%5Bdatacolumns%5D.apply(pdtonum)%0A            # Make timestamp unique and use mean of values on duplicates%0A            data = data.groupby(self.csvrequest.dtfield, as_index=False).mean()%0A            # Remove empty rows%0A            data = data.dropna(axis='rows', how='all', subset=datacolumns)%0A            # Remove timestamp column from data
 %0A   
@@ -2022,16 +2022,79 @@
 tfield%5D%0A
+            data = data.drop(columns=self.csvrequest.dtfield)%0A%0A
         
@@ -2167,33 +2167,24 @@
 m(timestamp)
-.dropna()
 %0A           
@@ -2334,33 +2334,24 @@
 m(timestamp)
-.dropna()
 %0A           
@@ -2501,33 +2501,24 @@
 m(timestamp)
-.dropna()
 %0A           
@@ -2675,17 +2675,8 @@
 amp)
-.dropna()
 %0A   
@@ -2731,32 +2731,154 @@
 p, unit = 'ns')%0A
+            elif dtformat == '%3Cinfer%3E':%0A                timestamp = pd.to_datetime(timestamp, infer_datetime_format=True)%0A
             else
@@ -2951,16 +2951,56 @@
 tformat)
+%0A%0A            # Convert timestamp to UTC
 %0A       
@@ -3194,135 +3194,8 @@
 p%5D)%0A
-            data = data.drop(columns=self.csvrequest.dtfield)%0A%0A        # Coerce all columns to numeric and remove empty columns
 %0A   
@@ -3215,23 +3215,8 @@
 ata.
-apply(pdtonum).
 drop
@@ -3249,59 +3249,8 @@
 l')%0A
-        data = data.dropna(axis='rows', how='all')%0A
     

Make sure the timestamp for data is unique. When duplicated timestamps are detected, use the mean value for the data