import Cheerio from 'cheerio';
import goodGuyHttp from 'good-guy-http';
import moment from 'moment';
import url from 'url';

const mediaLandingUrl = 'https://www.mediastatements.wa.gov.au/Pages/Portfolios/Mines-and-Petroleum.aspx';

const goodGuy = goodGuyHttp({
  headers: {
    'User-Agent': 'Department of Commerce Intranet - request',
  },
  proxy: 'http://proxy.bedrock.mft.wa.gov.au:8080',
  timeout: 15000,
});

exports.getMinisterials = async function getMinisterials(request, reply) {
  // Fetch the media statements landing page and wait for it to return
  const landingResponse = await goodGuy(mediaLandingUrl);
  // Load the HTML body into Cheerio
  const $ = Cheerio.load(landingResponse.body.toString());
  // Extract the data from the table of links
  const landingPage = $('tr > td', 'div.cs-rollup-content > table');
  // Get an array of the link elements
  const statementLinks = landingPage.find('a');
  // Create an array for holding the relative URLs from these links
  const linkPartials = [];
  // For each link element, extract the href attribute
  statementLinks.each((idx, elem) => {
    linkPartials[idx] = $(elem).attr('href');
  });
  // Create a new array holding the full URLs to each statement,
  // using url.resolve to make them
  const fullUrls = linkPartials.map((link, index) => {
    const fullUrl = url.resolve(mediaLandingUrl, link);
    // console.log(`Full URL ${index} = ${fullUrl}`);
    return fullUrl;
  });
  // Create an array of objects containing data extracted from
  // each media statement
  const statements = await fullUrls.map(async (link) => {
    // Fetch the statement page and wait for it to return
    const statementResponse = await goodGuy(link);
    // Load the statement body into Cheerio
    const $c = await Cheerio.load(statementResponse.body.toString());
    // Extract the article content element
    const article = await $c('div#article');
    // Extract the title text
    const title = article.find('h1').text();
    // Extract the raw date created text
    const rawDateString = article.find($c('div.newsCreatedDate')).text().trim();
    // Parse the date text into a proper Date object using moment.js and
    // the known formatting string
    const dateParsed = moment(rawDateString, 'D/MM/YYYY H:mm A');
    // Create a nice date string for use in the feed object
    const dateString = moment(dateParsed).format('dddd, D MMMM YYYY');
    // Create a more useful Unix epoch date (in seconds) for the feed object
    const dateUnix = moment(dateParsed).format('X');
    // Extract the media statement body
    const contentHtml = article.find('div.article-content');
    // Get rid of the leading <ul> element from the article HTML
    const contentHtmlCleaned = contentHtml.html().replace(/<ul>.*<\/ul>/, '');
    // console.log(contentHtmlCleaned);
    // Convert the HTML string from above back into a Cheerio object so
    // it can be further cleaned up
    const contents = Cheerio.load(contentHtmlCleaned).text().trim().replace(/^Page Content/, '');
    // const rawContents = contentHtmlCleaned.trim().replace(/^Page Content/, '');
    // const contents = Cheerio.load(rawContents).text();
    // Return nice media statement data object
    return {
      url: link,
      dateString,
      dateUnix,
      title,
      contents,
    };
  });
  // Wait for the all the Promise objects in this array to resolve
  const statementData = Promise.all(statements);
  // console.log('Statements obj = ');
  // console.dir(await statementData);
  // Hapi reply call to send back object data serialised to JSON
  reply(await statementData);
};

@@ -350,49 +350,30 @@
 xy: 
-'http://proxy.bedrock.mft.wa.gov.au:8080'
+process.env.HTTP_PROXY
 ,%0A  
@@ -1282,15 +1282,15 @@
 link
-, index
+Partial
 ) =%3E
@@ -1300,23 +1300,14 @@
     
-const fullUrl =
+return
  url
@@ -1340,83 +1340,16 @@
 link
-);%0A    // console.log(%60Full URL $%7Bindex%7D = $%7BfullUrl%7D%60);%0A    return fullUrl
+Partial)
 ;%0A  
@@ -1488,20 +1488,23 @@
 (async (
-link
+fullUrl
 ) =%3E %7B%0A 
@@ -1604,20 +1604,23 @@
 goodGuy(
-link
+fullUrl
 );%0A    /
@@ -2491,477 +2491,230 @@
 body
-%0A    const contentHtml = article.find('div.article-content');%0A    // Get rid of the leading %3Cul%3E element from the article HTML%0A    const contentHtmlCleaned = contentHtml.html().replace(/%3Cul%3E.*%3C%5C/ul%3E/, '');%0A    // console.log(contentHtmlCleaned);%0A    // Convert the HTML string from above back into a Cheerio object so%0A    // it can be further cle
+ text,%0A    // skipping the %3Cul%3E element at the top and the %22Page Content%22 target link%0A    const contentHtml = article.find('div.article-content p');%0A    // Convert the Cheerio object to text 
 an
-e
 d 
-up%0A    const contents = Cheerio.load(contentHtmlCleaned).text().trim().replace(/%5EPage Content/, '');
+trim whitespace
 %0A   
- //
  const 
-rawC
+c
 onte
@@ -2734,107 +2734,20 @@
 Html
-Cleaned.trim().replace(/%5EPage Content/, '');%0A    // const contents = Cheerio.load(rawContents).text
+.text().trim
 ();%0A
@@ -2817,20 +2817,23 @@
    url: 
-link
+fullUrl
 ,%0A      

Make article text extraction cleaner by just getting content of <p> elements