#!/usr/bin/env python

from bs4 import BeautifulSoup
import json
import requests
#import urllib
import sys

#URI of the data source
URI = sys.argv[1] if len(sys.argv) > 1 else 'https://arc.lib.montana.edu/ivan-doig/about.php'
#URI = 'https://arc.lib.montana.edu/ivan-doig/about.php'
#if len(sys.argv) > 1:
    #URI = sys.argv[1]

def parseSource(uri):
    request = requests.get(uri, headers={'User-Agent' : 'jasonclark.info indexing bot'})
    #request = urllib.urlopen(uri).read()

    #check for HTTP codes other than 200
    if request.status_code != 200:
        print('Status:', request.status_code, 'Problem with the request. Exiting.')
        exit()

    soup = BeautifulSoup(request.text, 'html.parser')
    #soup = BeautifulSoup(request, 'html.parser')
    #print soup.findAll('a')
    #print soup.prettify()
    #print(soup.get_text())

    title = soup.title.string

    print ('Page Title: \n' + title)
    print ('Page URL: \n' + uri)

    #set empty list for about json values
    aboutList = [] 

    #for link in soup.find_all('a', attrs={'property':'about'}):
    for link in soup.find_all(property='about'):
        #print('about data: \n' + link.get('href'))
        tagValue = link.string.strip('\r\n\t')
        print('about data: \n' + tagValue)
        aboutList.append((tagValue, len(tagValue)))
	
    with open('json-schema-about.txt', 'w') as outfile:
        json.dump(aboutList, outfile, indent = 4)

    #set empty list for type json values
    typeList = []

    #for dblink in soup.find_all('link', attrs={'property':'additionalType'}):
    for dblink in soup.find_all(property='additionalType'):
        #print('additionalType data: \n' + dblink.get('resource'))
        print('additionalType data: \n' + dblink.string)
	#print json.dumps(dblink.string, indent = 4)
        typeList.append((dblink.string, len(dblink.string)))

    with open('json-schema-types.txt', 'w') as outfile:
        json.dump(typeList, outfile, indent = 4)

showResult = parseSource(URI)
print showResult
print 'JSON files created.'

@@ -1292,18 +1292,37 @@
 end(
-(tagValue,
+%7B%22about%22: tagValue, %22length%22:
  len
@@ -1331,17 +1331,17 @@
 agValue)
-)
+%7D
 )%0A%09%0A    
@@ -1836,17 +1836,25 @@
 .append(
-(
+%7B%22type%22: 
 dblink.s
@@ -1860,16 +1860,25 @@
 string, 
+%22length%22:
 len(dbli
@@ -1887,17 +1887,17 @@
 .string)
-)
+%7D
 )%0A%0A    w

add brackets and keys to JSON output