#!/usr/bin/env node

'use strict';

const BBPromise = require('bluebird');
const fs = require("fs");
const preq = require('preq');
const path = require('path');

const BLACKLIST = require(path.join(__dirname, '../etc/feed/blacklist'));
const SPECIAL = 'Special:';
const SPECIAL2 = 'special:';

// Will be set later
let lang;
let topMonthlyPageViews;
let topPagesFile;
let parsoidBaseUri;

const fixTitleForRequest = (pageTitle) => {
    return encodeURIComponent(pageTitle);
};

const uriForParsoid = (pageTitle) => {
    return `${parsoidBaseUri}/${fixTitleForRequest(pageTitle)}`;
};

const writePages = (myPages) => {
    const logger = fs.createWriteStream(topPagesFile, { flags: 'w' });
    logger.write(`{ "items": [\n`);
    myPages.forEach((page, index, array) => {
        if (page) {
            const comma = (index < array.length - 1) ? ',' : '';
            const title = page.title && page.title.replace(/"/g, '\\"');
            logger.write(`  { "title": "${title}", "rev": "${page.rev}" }${comma}\n`);
        }
    });
    logger.write(`]}\n`);
    logger.end();
};

const processOnePage = (page) => {
    process.stdout.write('.');
    return preq.get({ uri: uriForParsoid(page.title) })
    .then((rsp) => {
        return BBPromise.delay(1, rsp);
    }).then((rsp) => {
        if (rsp.status !== 200) {
            if (rsp.status === 302) { // redirect through 302
                page.title = rsp.headers.location;
                return processOnePage(page);
            }
            process.stderr.write(` WARNING: skipping parsoid for ${page.title}!`);
            return BBPromise.resolve();
        }
        const contentLocation = rsp.headers['content-location'];
        if (contentLocation) {
            page.title = decodeURIComponent(
                contentLocation.substring(contentLocation.lastIndexOf('/') + 1));
        }
        const etag = rsp.headers.etag;
        const revMatch = /"(\S+?)"/m.exec(etag);
        page.rev = revMatch[1];
        return page;
    }).catch((err) => {
        if (err.status === 504) {
            process.stderr.write(` Timeout for ${page.title}: ${uriForParsoid(page.title)}! `);
            // time out encountered: wait a few seconds and try again
            return BBPromise.delay(2000).then(() => processOnePage(page));
        } else {
            process.stderr.write(`
ERROR getting metadata for ${page.title}: ${err.status}: ${err.body.detail}
`);
        }
    });
};

const getETags = (myPages) => {
    return BBPromise.map(myPages, (page) => {
        return processOnePage(page);
    }, { concurrency: 1 })
    .then((myPages) => {
        writePages(myPages);
    });
};

const getTopPageViews = () => {
    return preq.get({ uri: topMonthlyPageViews })
    .then((rsp) => {
        return rsp.body.items[0].articles.filter((article) => {
            const title = article.article;
            return (title.indexOf(SPECIAL) !== 0 && title.indexOf(SPECIAL2) !== 0
                && !BLACKLIST.includes(title));
        }).map((article) => {
            return { "title": article.article };
        });
    }).catch((err) => {
        process.stderr.write(`ERROR: could not get top monthly page views: ${err}`);
    }).then((myPages) => {
        getETags(myPages);
    });
};

// MAIN
const arg = process.argv[2];
if (arg) {
    lang = arg;
    topMonthlyPageViews = `https://wikimedia.org/api/rest_v1/metrics/pageviews/top/${lang}.wikipedia/all-access/2017/06/all-days`; // eslint-disable-line max-len
    topPagesFile = path.join(__dirname, `../private/top-pages/top-pages.${lang}.json`);
    parsoidBaseUri = `https://${lang}.wikipedia.org/api/rest_v1/page/html`;

    getTopPageViews();
}

@@ -285,16 +285,151 @@
 ecial:';
+%0Aconst PROJECT = 'wikipedia';%0Aconst YEAR = '2017';%0Aconst MONTH = '06';%0Aconst OUTPUT_DIR = path.join(__dirname, %60../private/top-pages%60);
 %0A%0A// Wil
@@ -482,24 +482,22 @@
 ws;%0Alet 
-topPages
+output
 File;%0Ale
@@ -787,24 +787,22 @@
 eStream(
-topPages
+output
 File, %7B 
@@ -3550,25 +3550,26 @@
 $%7Blang%7D.
-wikipedia
+$%7BPROJECT%7D
 /all-acc
@@ -3576,15 +3576,24 @@
 ess/
-2017/06
+$%7BYEAR%7D/$%7BMONTH%7D
 /all
@@ -3639,16 +3639,14 @@
     
-topPages
+output
 File
@@ -3662,41 +3662,21 @@
 oin(
-__dirname, %60../private/top-pages/
+OUTPUT_DIR, %60
 top-
@@ -3735,25 +3735,26 @@
 $%7Blang%7D.
-wikipedia
+$%7BPROJECT%7D
 .org/api

make top-pages script easier to configure