# Copyright 2018 Alex K (wtwf.com) All rights reserved.

"""
Find the search url (and the suggest url) for a site
gold standard (even if it's php and downloads the file)

http://www.gutenberg.org/
http://www.gutenberg.org/w/opensearch_desc.php
http://www.gutenberg.org/w/api.php?action=opensearch&search=arctic&namespace=0|4

also should support ld+json (doesn't seem to have suggest support)
https://developers.google.com/search/docs/data-types/sitelinks-searchbox

https://www.chewy.com/

other sites:
https://www.costco.com/
https://www.airbnb.com/opensearch.xml
https://www.crunchbase.com/opensearch.xml?version=2
https://www.labnol.org/
https://www.diigo.com/search/open_search
https://community.dremio.com/
https://domains.google/#/
https://earlyretirementnow.com/osd.xml
https://www.flickr.com/opensearch.xml

"""

__author__ = 'wtwf.com (Alex K)'

import json
import logging
import re
import urlparse
import HTMLParser
import xml.etree.ElementTree

import google.appengine.api.urlfetch

from wtwf import wtwfhandler


class FindHrefParser(HTMLParser.HTMLParser):

  def reset(self):
    HTMLParser.HTMLParser.reset(self)
    self.hrefs = []

  def handle_starttag(self, tag, attrs):
    if 'href' in attrs:
      self.hrefs.append(attrs['href'])


class FindOpensearchHandler(wtwfhandler.WtwfHandler):

  def get(self):
    url = self.request.get('url')
    base = urlparse.urljoin(url, '/')

    res = google.appengine.api.urlfetch.fetch(base)

    content = res.content

    # first try and find opensearch
    opensearch_re = re.compile(r'<[^>]*opensearchdescription\+xml[^>]*>')
    match = opensearch_re.search(content)
    if match:
      parser = FindHrefParser()
      parser.feed(match.group(0))
      if len(parser.hrefs) > 0:
        href = parser.hrefs[0]
        res = google.appengine.api.urlfetch.fetch(urlparse.urljoin(base, href))
        content = res.content
        # xml parse that shit
        root = xml.etree.ElementTree.fromstring(content)
        # get the Url elements (url, URL too)

        # if it's text/html it's the search url, get template, replace {searchTerms} with %s
        # make sure it doesn't have &amp;

        # if it's application/x-suggestions+json then it's the suggest url

        # if it only has application/x-suggestions+xml then that's messed up


    # second try and find ld+json

@@ -1512,16 +1512,319 @@
 earch%0A  
+  opensearch_url = getOpenSearchUrl(res.content)%0A    if opensearch_url:%0A      res = google.appengine.api.urlfetch.fetch(urlparse.urljoin(base, opensearch_url))%0A      search_and_suggest = getSearchAndSuggestFromOpenSearchXml(res.content)%0A%0A%0A    # second try and find ld+json%0A%0A%0Adef getOpenSearchUrl(page):%0A
   opense
@@ -1887,18 +1887,16 @@
 %5E%3E%5D*%3E')%0A
-  
   match 
@@ -1918,27 +1918,22 @@
 .search(
-content)%0A  
+page)%0A
   if mat
@@ -1936,26 +1936,24 @@
  match:%0A    
- 
- 
 parser = Fin
@@ -1966,18 +1966,16 @@
 arser()%0A
-  
     pars
@@ -1998,18 +1998,16 @@
 oup(0))%0A
-  
     if l
@@ -2038,16 +2038,14 @@
     
-  href =
+return
  par
@@ -2061,124 +2061,56 @@
 %5B0%5D%0A
-        res = google.appengine.api.urlfetch.fetch(urlparse.urljoin(base, href))%0A        content = res.content%0A      
+%0Adef getSearchAndSuggestFromOpenSearchXml(xml):%0A
   # 
@@ -2129,22 +2129,16 @@
 at shit%0A
-      
   root =
@@ -2175,23 +2175,47 @@
 ing(
-content)%0A      
+xml)%0A  search = None%0A  suggest = None%0A%0A
   # 
@@ -2250,23 +2250,36 @@
 RL too)%0A
-%0A
   
-    
+root.find('url')%0A%0A
   # if i
@@ -2357,22 +2357,16 @@
 with %25s%0A
-      
   # make
@@ -2394,22 +2394,16 @@
  &amp;%0A%0A
-      
   # if i
@@ -2464,22 +2464,16 @@
 st url%0A%0A
-      
   # if i
@@ -2541,38 +2541,75 @@
 p%0A%0A%0A
+%0A
   
-  # second try and find ld+json
+if search or suggest:%0A    return %7Bsearch: search, suggest: suggest%7D
 %0A

wip. I should be able to make this testable