#!/usr/bin/env node

const fs = require('fs');
const url = require('url');
const util = require('util');
const request = require('request-promise-native');
const argv = require('yargs')
    .options({
        year: {
            type: 'number',
            describe: 'election year',
            default: Math.ceil(new Date().getFullYear() / 2)* 2,
        },
    })
    .strict(true)
    .argv;
const Browser = require('zombie');
const browser = new Browser({waitDuration: '30s'});
const electionYear = argv.year;

// Force 5s pause between requests
browser.pipeline.addHandler(function (browser, request) {
    // Log the response body
    console.log(request.method, request.url);
    return new Promise(resolve => setTimeout(resolve, 5000));
});

writeTransactionCsv('expenditures').catch(err => console.error(err));

function writeCommitteeCsv() {
    const file =__dirname + '/committees.csv';
    return browser.visit('https://efiling.ocf.dc.gov/Disclosure')
        .then(() => browser.select('#FilerTypeId', 'Principal Campaign Committee'))
        .then(() => browser.select('#ElectionYear', electionYear))
        .then(() => browser.click('#btnSubmitSearch'))
        .then(getCsv)
        .then(csv => fs.writeFileSync(file, csv, {encoding: 'utf-8'}))
        .then(() => process.exit());
}

function writeTransactionCsv(type) {
    const file =__dirname + '/' + type + '.csv';
    return browser.visit('https://efiling.ocf.dc.gov/ContributionExpenditure')
        .then(() => browser.select('#FilerTypeId', 'Principal Campaign Committee'))
        .then(() => browser.click('#' + type))
        .then(() => browser.click('#accordionpanel4 a'))
        .then(() => browser.fill('#FromDate', '01/01/' + (electionYear - 2)))
        .catch(function (err) {
            if (err.message === "Cannot read property 'settings' of undefined") {
                return null;
            }
            throw err;
        })
        .then(() => browser.click('#btnSubmitSearch'))
        .then(getCsv)
        .then(csv => fs.writeFileSync(file, csv, {encoding: 'utf-8'}));
}

function getCsv() {
    return browser.click('#divExportDropdown')
        .then(function () {
            const link = browser.query('#exportDropDown > li:last-child > a');
            const csvUrl = url.resolve(browser.location.href, link.getAttribute('href'));
            browser.tabs.closeAll();
            // Annoyingly I had to go through all this using request because if I try just using zombie to
            // click on the link it never comes back if the response is big
            const jar = request.jar();
            for (const cookie of browser.cookies) {
                jar.setCookie(request.cookie(cookie.toString()), csvUrl);
            }
            return request({url: csvUrl, jar});
        })
        .then(function (body) {
            // Convert UTF-16 to UTF-8
            const csv = new util.TextDecoder('utf-16').decode(Buffer.from(body));
            // Fix line endings and delete first line (which is not CSV)
            return csv.replace(/\r\n/g, '\n')
                .replace(/^.+\n/, '');
        });
}

@@ -361,27 +361,735 @@
     
-%7D)%0A    .strict(true
+    committees: %7B%0A            type: 'boolean',%0A            describe: 'download committees',%0A        %7D,%0A        contributions: %7B%0A            type: 'boolean',%0A            describe: 'download contributions',%0A        %7D,%0A        expenditures: %7B%0A            type: 'boolean',%0A            describe: 'download expenditures',%0A        %7D,%0A        all: %7B%0A            type: 'boolean',%0A            describe: 'download committees, contributions, and expenditures',%0A        %7D,%0A    %7D)%0A    .strict(true)%0A    .check(function (argv, options) %7B%0A        if (!(argv.committees %7C%7C argv.contributions %7C%7C argv.expenditures %7C%7C argv.all)) %7B%0A            throw new Error('Must specify at least one data set to download');%0A        %7D%0A        return true;%0A    %7D
 )%0A  
@@ -1215,16 +1215,103 @@
 gv.year;
+%0Aif (argv.all) %7B%0A    argv.committees = argv.contributions = argv.expenditures = true;%0A%7D
 %0A%0A// For
@@ -1330,16 +1330,21 @@
 between 
+main 
 requests
@@ -1410,146 +1410,334 @@
     
-// Log the response body%0A    console.log(request.method, request.url);%0A    return new Promise(resolve =%3E setTimeout(resolve, 5000));%0A%7D);%0A%0A
+return new Promise(resolve =%3E setTimeout(resolve, 5000));%0A%7D);%0A%0Alet promise = Promise.resolve();%0Aif (argv.committees) %7B%0A    promise = promise.then(writeCommitteeCsv);%0A%7D%0Aif (argv.contributions) %7B%0A    promise = promise.then(() =%3E writeTransactionCsv('contributions'));%0A%7D%0Aif (argv.expenditures) %7B%0A    promise = promise.then(() =%3E 
 writ
@@ -1767,16 +1767,28 @@
 itures')
+);%0A%7D%0Apromise
 .catch(e
@@ -1812,16 +1812,48 @@
 or(err))
+%0A    .then(() =%3E process.exit())
 ;%0A%0Afunct
@@ -2139,16 +2139,27 @@
 tionYear
+.toString()
 ))%0A     
@@ -2304,44 +2304,8 @@
 '%7D))
-%0A        .then(() =%3E process.exit())
 ;%0A%7D%0A

allow downloading of any data set; add more options