/* jshint esversion: 6, node: true */
const Fs = require('fs');
const Semaphore = require('saferphore');
const nThen = require('nthen');

const hashesFromPinFile = (pinFile, fileName) => {
    var pins = {};
    pinFile.split('\n').filter((x)=>(x)).map((l) => JSON.parse(l)).forEach((l) => {
        switch (l[0]) {
            case 'RESET': {
                pins = {};
                //jshint -W086
                // fallthrough
            }
            case 'PIN': {
                l[1].forEach((x) => { pins[x] = 1; });
                break;
            }
            case 'UNPIN': {
                l[1].forEach((x) => { delete pins[x]; });
                break;
            }
            default: throw new Error(JSON.stringify(l) + '  ' + fileName);
        }
    });
    return Object.keys(pins);
};

const sizeForHashes = (hashes, dsFileStats) => {
    let sum = 0;
    hashes.forEach((h) => {
        const s = dsFileStats[h];
        if (typeof(s) !== 'object' || typeof(s.size) !== 'number') {
            //console.log('missing ' + h + '  ' + typeof(s));
        } else {
            sum += s.size;
        }
    });
    return sum;
};

const sema = Semaphore.create(20);

let dirList;
const fileList = [];
const dsFileStats = {};
const out = [];
const pinned = {};

nThen((waitFor) => {
    Fs.readdir('./datastore', waitFor((err, list) => {
        if (err) { throw err; }
        dirList = list;
    }));
}).nThen((waitFor) => {
    dirList.forEach((f) => {
        sema.take((returnAfter) => {
            Fs.readdir('./datastore/' + f, waitFor(returnAfter((err, list2) => {
                if (err) { throw err; }
                list2.forEach((ff) => { fileList.push('./datastore/' + f + '/' + ff); });
            })));
        });
    });
}).nThen((waitFor) => {

    Fs.readdir('./blob', waitFor((err, list) => {
        if (err) { throw err; }
        dirList = list;
    }));
}).nThen((waitFor) => {
    dirList.forEach((f) => {
        sema.take((returnAfter) => {
            Fs.readdir('./blob/' + f, waitFor(returnAfter((err, list2) => {
                if (err) { throw err; }
                list2.forEach((ff) => { fileList.push('./blob/' + f + '/' + ff); });
            })));
        });
    });
}).nThen((waitFor) => {
    fileList.forEach((f) => {
        sema.take((returnAfter) => {
            Fs.stat(f, waitFor(returnAfter((err, st) => {
                if (err) { throw err; }
                dsFileStats[f.replace(/^.*\/([^\/\.]*)(\.ndjson)?$/, (all, a) => (a))] = st;
            })));
        });
    });
}).nThen((waitFor) => {
    Fs.readdir('./pins', waitFor((err, list) => {
        if (err) { throw err; }
        dirList = list;
    }));
}).nThen((waitFor) => {
    fileList.splice(0, fileList.length);
    dirList.forEach((f) => {
        sema.take((returnAfter) => {
            Fs.readdir('./pins/' + f, waitFor(returnAfter((err, list2) => {
                if (err) { throw err; }
                list2.forEach((ff) => { fileList.push('./pins/' + f + '/' + ff); });
            })));
        });
    });
}).nThen((waitFor) => {
    fileList.forEach((f) => {
        sema.take((returnAfter) => {
            Fs.readFile(f, waitFor(returnAfter((err, content) => {
                if (err) { throw err; }
                const hashes = hashesFromPinFile(content.toString('utf8'), f);
                const size = sizeForHashes(hashes, dsFileStats);
                if (process.argv.indexOf('--unpinned') > -1) {
                    hashes.forEach((x) => { pinned[x] = 1; });
                } else {
                    out.push([f, Math.floor(size / (1024 * 1024))]);
                }
            })));
        });
    });
}).nThen(() => {
    if (process.argv.indexOf('--unpinned') > -1) {
        const ot = process.argv.indexOf('--olderthan');
        let before = 0;
        if (ot > -1) {
            before = new Date(process.argv[ot+1]);
            if (isNaN(before)) {
                throw new Error('--olderthan error [' + process.argv[ot+1] + '] not a valid date');
            }
        }
        Object.keys(dsFileStats).forEach((f) => {
            if (!(f in pinned)) {
                if ((+dsFileStats[f].mtime) >= before) { return; }
                console.log("./datastore/" + f.slice(0,2) + "/" + f + ".ndjson " +
                    dsFileStats[f].size + " " + (+dsFileStats[f].mtime));
            }
        });
    } else {
        out.sort((a,b) => (a[1] - b[1]));
        out.forEach((x) => { console.log(x[0] + '  ' + x[1] + ' MB'); });
    }
});

@@ -2409,32 +2409,65 @@
  %7B throw err; %7D%0A
+                st.filename = f;%0A
                 
@@ -3853,17 +3853,24 @@
 efore = 
-0
+Infinity
 ;%0A      
@@ -4106,137 +4106,591 @@
     
-Object.keys(dsFileStats).forEach((f) =%3E %7B%0A            if (!(f in pinned)) %7B%0A                if ((+dsFileStats%5Bf%5D.mtime) %3E= before
+const bot = process.argv.indexOf('--blobsolderthan');%0A        let blobsbefore = before;%0A        if (bot %3E -1) %7B%0A            blobsbefore = new Date(process.argv%5Bbot+1%5D);%0A            if (isNaN(blobsbefore)) %7B%0A                throw new Error('--blobsolderthan error %5B' + process.argv%5Bbot+1%5D + '%5D not a valid date');%0A            %7D%0A        %7D%0A        Object.keys(dsFileStats).forEach((f) =%3E %7B%0A            if (!(f in pinned)) %7B%0A                const isBlob = dsFileStats%5Bf%5D.filename.indexOf('.ndjson') === -1;%0A                if ((+dsFileStats%5Bf%5D.mtime) %3E= ((isBlob) ? blobsbefore : before)
 ) %7B 
@@ -4731,58 +4731,63 @@
 log(
-%22./datastore/%22 + f.slice(0,2)
+dsFileStats%5Bf%5D.filename
  + %22
-/
+ 
 %22 + 
-f + %22.ndjson
+dsFileStats%5Bf%5D.size + %22
  %22 +
@@ -4810,36 +4810,8 @@
     
- dsFileStats%5Bf%5D.size + %22 %22 +
  (+d

Update pinneddata to work on blobs too