"""Script to join all the chisquared part files into a sql database."""
import argparse
import glob as glob
import sys

import pandas as pd
import sqlalchemy as sa

def _parser():
    """Take care of all the argparse stuff.

    :returns: the args
    """
    parser = argparse.ArgumentParser(description='Wavelength Calibrate CRIRES \
                                    Spectra')
    parser.add_argument('pattern', help='Pattern')
    parser.add_argument('-s', '--suffix', help='Suffix to add to database name.')
    parser.add_argument('-v', '--verbose', help='Turn on Verbose.', action="store_true")
    # parser.add_argument('-s', '--sort', help='Sort by column.', default='chi2')
    # parser.add_argument('--direction', help='Sort direction.',
    #                     default='ascending', choices=['ascending','decending'])
    parser.add_argument("-r", '--remove', action="store_true",
                        help='Remove original files after joining (default=False).')
    return parser.parse_args()


def sql_join(pattern, suffix="", verbose=True, remove=False):
    """Join data files with names that match a given pattern.

    Inputs
    ------
    pattern: str
        Regex to match with glob. Part before "_part" is prefix to databse.
    suffix: str
        Identifier to add before .db.
    verbose: bool (default True)
        Verbose level

    Returns
    -------
    None
    Saves file to sql database

    http://pythondata.com/working-large-csv-files-python/
    """
    number_of_files = sum(1 for _ in glob.iglob(pattern))
    if verbose:
        print("Concatinating {} files.".format(number_of_files))

    # Get frist part of name
    prefix = next(glob.iglob(pattern)).split("_part")[0]
    print(prefix, suffix)
    database_name = 'sqlite:///{0}{1}.db'.format(prefix, suffix)
    engine = sa.create_engine(database_name)
    print("csv_database =", engine, type(engine))

    chunksize = 100000
    i = 0
    j = 1
    for f in glob.iglob(pattern):

        if "[" in f:
            n = f.split("[")[-1]
            n = n.split("]")[0]
            teff, logg, feh = n.split("_")
            print("host params", teff, logg, feh)
            host_flag = True
        else:
            host_flag = False

        for df in pd.read_csv(f, chunksize=chunksize, iterator=True):
            # print("chunk number = {}".format(i))
            if host_flag:
                df["teff_1"] = teff
                df["logg_1"] = logg
                df["feh_1"] = feh
                df = df.rename(columns={c: c.replace(' ', '').lower() for c in df.columns})
                df.index += j
                i += 1
                df.to_sql('chi2_table', engine, if_exists='append')
                j = df.index[-1] + 1
                if verbose:
                    print("indicies = ", i, j)

    if verbose:
            print("Saved results to {}.".format(database_name))

    if remove:
        print("Removing original files.")
        raise NotImplementedError
        # (subprocess.call("rm ") for f in glob.iglob(pattern))
        
    return 0

if __name__ == '__main__':
    args = vars(_parser())
    opts = {k: args[k] for k in args}

    sys.exit(sql_join(**opts))

@@ -101,16 +101,44 @@
 as glob%0A
+import os%0Aimport subprocess%0A
 import s
@@ -186,16 +186,17 @@
  as sa%0A%0A
+%0A
 def _par
@@ -535,16 +535,30 @@
 e name.'
+, default=None
 )%0A    pa
@@ -899,17 +899,15 @@
 t(%22-
-r
+m
 %22, '--
-re
 move
@@ -960,21 +960,167 @@
   help='
-Remov
+Move original files after joining (default=False).')%0A    parser.add_argument(%22-r%22, '--remove', action=%22store_true%22,%0A                        help='Delet
 e origin
@@ -1223,18 +1223,20 @@
  suffix=
-%22%22
+None
 , verbos
@@ -1242,16 +1242,28 @@
 se=True,
+ move=False,
  remove=
@@ -1825,24 +1825,67 @@
 of_files))%0A%0A
+    if suffix is None:%0A        suffix = %22%22%0A
     # Get fr
@@ -3055,16 +3055,279 @@
  i, j)%0A%0A
+        if move:%0A            f_split = os.path.split(f)%0A            new_f = os.path.join(f_split%5B0%5D, %22processed_cvs%22, f_split%5B1%5D)%0A            os.makedirs(os.path.dirname(new_f), exist_ok=True)%0A            subprocess.call(%22mv %7B%7D %7B%7D%22.format(f, new_f), shell=True)%0A%0A
     if v
@@ -3554,24 +3554,16 @@
 ttern))%0A
-        
 %0A    ret
@@ -3569,16 +3569,17 @@
 turn 0%0A%0A
+%0A
 if __nam

Add database suffix with None default. Move files once put into database.