#!/usr/bin/env python3
#
# link-validator.py
# Brandon Amos <http://bamos.io>

import argparse
import re
from urllib.request import urlopen
from urllib.parse import urljoin,urldefrag
from bs4 import BeautifulSoup, SoupStrainer

baseUrl = None
invalidWikiPages = []
visitedUrls = []

def validate(url):
  if url in visitedUrls: return

  visitedUrls.append(url)
  try:
    content = urlopen(url).read().decode("utf8")
  except:
    # Assume the content is binary.
    return

  wikiUrls = []
  invalidUrls = []
  for externalUrl in BeautifulSoup(content, parse_only=SoupStrainer('a')):
    if externalUrl.has_attr('href'):
      fullExternalUrl = urljoin(url, urldefrag(externalUrl['href']).url)
      if baseUrl in fullExternalUrl and \
          not fullExternalUrl.endswith('/_history'):
        if externalUrl.has_attr('class') and 'absent' in externalUrl['class']:
          invalidUrls.append(fullExternalUrl)
        else:
          wikiUrls.append(fullExternalUrl)

  if len(invalidUrls) > 0:
    invalidWikiPages.append((url, invalidUrls))

  for wikiUrl in wikiUrls:
    if wikiUrl not in visitedUrls:
      validate(wikiUrl)

if __name__ == '__main__':
  parser = argparse.ArgumentParser()
  parser.add_argument('url', type=str,
      help='The base link to the GitHub Wiki to scrape. ' +
      'Example: http://github.com/bamos/github-wiki-link-validator/wiki')
  args = parser.parse_args()

  baseUrl = args.url
  validate(args.url)

  if len(invalidWikiPages) == 0:
    print("No invalid links found.")
  else:
    for url, invalidUrls in invalidWikiPages:
      print("+ " + url)
      for invalidUrl in invalidUrls:
        print("  + " + invalidUrl)

@@ -509,120 +509,407 @@
 %5D%0A  
-for externalUrl in BeautifulSoup(content, parse_only=SoupStrainer('a')):%0A    if externalUrl.has_attr('href'):%0A  
+# This may see redundant, but without the %60.find_all('a')%60, soup will also%0A  # contain the %60DocType%60 element which does not have an %60href%60 attribute.%0A  # See %3Chttp://stackoverflow.com/questions/17943992/beautifulsoup-and-soupstrainer-for-getting-links-dont-work-with-hasattr-returni%3E.%0A  soup = BeautifulSoup(content, parse_only=SoupStrainer('a', href=True)).find_all('a')%0A  for externalUrl in soup:%0A
     
@@ -971,26 +971,24 @@
 ref'%5D).url)%0A
-  
     if baseU
@@ -1023,18 +1023,16 @@
         
-  
 not full
@@ -1066,18 +1066,16 @@
 tory'):%0A
-  
       if
@@ -1151,18 +1151,16 @@
         
- 
- 
 invalidU
@@ -1193,26 +1193,22 @@
 )%0A      
-  
 else:%0A
-  
         

fix to skip over the DocType element