import argparse
import tqdm


def pairwise(iterable):
    a = iter(iterable)
    return zip(a, a)


def remove_dups(history):
    from_end = 1
    total_dups = 0
    with tqdm.tqdm(total=len(history)-1) as progress:
        while from_end < len(history):
            target = len(history) - from_end
            command = history[target][0]
            dups = 0
            for i in range(target - 1, -1, -1):
                if history[i][0] == command:
                    del history[i]
                    dups += 1
                    progress.update(1)
            total_dups += dups
            from_end += 1
            progress.update(1)
    print("Removed %d duplicates" % total_dups)
    return total_dups


def save_to_file(history, filename):
    filename += '.deduped'
    with open(filename, 'w') as f:
        for command, date in history:
            f.write('%s\n%s\n' % (date, command))
    print("Wrote to", filename)


def main(filename, minlen):
    with open(filename) as f:
        lines = f.read().splitlines()
        history = [(command.strip(), date)
                   for date, command in pairwise(lines)]
    if minlen:
        shorts = 0
        for i in range(len(history) - 1, -1, -1):
            if len(history[i][0]) < minlen:
                del history[i]
                shorts += 1
        print("Removed %d commands shorter than %d" % (shorts, minlen))
    remove_dups(history)
    save_to_file(history, filename)


if __name__ == '__main__':
    parser = argparse.ArgumentParser(
        description='Bash history file de-duplicator')
    parser.add_argument('-m', action="store", dest="minlen", type=int,
                        default=0, help='Minimum command length to save')
    parser.add_argument('history_file', action="store",
                        type=argparse.FileType('rt'))
    results = parser.parse_args()
    main(results.history_file.name, results.minlen)

@@ -151,24 +151,91 @@
 al_dups = 0%0A
+    print(%22Checking %25d commands for duplicates...%22 %25 len(history))%0A
     with tqd
@@ -1204,37 +1204,182 @@
     
-if minlen:%0A        shorts = 0
+print(%22Processing %25s with %25d commands%22 %25 (filename, len(history)))%0A    if minlen:%0A        shorts = 0%0A        print(%22Removing commands shorter than %25d characters...%22 %25 minlen)
 %0A   
@@ -1563,31 +1563,36 @@
 ommands 
-shorter than %25d
+%25d chars or shorter.
 %22 %25 (sho
@@ -1602,16 +1602,18 @@
 , minlen
+-1
 ))%0A    r
@@ -1632,16 +1632,83 @@
 istory)%0A
+    print(%22De-duplicated history has %25d commands.%22 %25 len(history))%0A
     save

Print extra progess info