import json
from urllib.parse import urljoin, urlparse, urlunparse
from concurrent.futures import ThreadPoolExecutor

import requests_cache
import lxml.html as html


BASE_URL = "http://na.finalfantasyxiv.com/lodestone/playguide/db/recipe/"

LANGUAGES = {
    "ja": "jp.finalfantasyxiv.com",
    "fr": "fr.finalfantasyxiv.com",
    "de": "de.finalfantasyxiv.com",
}

CLASSES = [
    "Carpenter",
    "Blacksmith",
    "Armorer",
    "Goldsmith",
    "Leatherworker",
    "Weaver",
    "Alchemist",
    "Culinarian",
]

LEVEL_RANGE = ["{0}-{1}".format(start, start + 4) for start in range(1, 60, 5)]
NUM_LEVEL_RANGES = len(LEVEL_RANGE)

session = requests_cache.CachedSession()
executor = ThreadPoolExecutor(max_workers=4)

def parse_recipe_links_page(base_url, r):
    tree = html.fromstring(r.text)
    rel_links = tree.xpath("//div/@data-ldst-href")
    links = [urljoin(base_url, str(rel_link)) for rel_link in rel_links]
    show_end = int(tree.xpath("//span[@class='show_end']/text()")[0])
    total = int(tree.xpath("//span[@class='total']/text()")[0])
    return links, show_end, total


def fetch_recipe_links_page(cls, level_range, page):
    params = {
        "category2": CLASSES.index(cls),
        "category3": level_range,
        "page": page,
    }

    return parse_recipe_links_page(BASE_URL, session.get(BASE_URL, params=params))

def fetch_recipe_links(cls):
    links = []

    for level_range in range(0, NUM_LEVEL_RANGES):
        page = 1
        while True:
            print("\rFetching {0} links... {1} page {2}".format(cls, LEVEL_RANGE[level_range], page), end="")
            page_links, show_end, total = fetch_recipe_links_page(cls, level_range, page)
            links += page_links
            if show_end < total:
                page += 1
            else:
                break

    print("\rFetching {0} links... done.".format(cls))

    return links


def switch_url_host(url, host):
    parsed_url = urlparse(url)
    return urlunparse((parsed_url.scheme, host, parsed_url.path, parsed_url.params, parsed_url.query, parsed_url.fragment))

def fetch_recipe(url):
    pages = {"en": executor.submit(session.get, url)}

    for lang in LANGUAGES:
        url = switch_url_host(url, LANGUAGES[lang])
        pages[lang] = executor.submit(session.get, url)

    tree = html.fromstring(pages["en"].result().text)
    recipe = {
        "name" : { "en": str(tree.xpath("//h2[contains(@class,'item_name')]/text()")[0]) },
        "level" : int(tree.xpath("//div[@class='recipe_level']/span/text()")[0]),
        "difficulty" : int(tree.xpath("//dl/dt[text()='Difficulty']/following-sibling::dd[1]/text()")[0]),
        "durability" : int(tree.xpath("//dl/dt[text()='Durability']/following-sibling::dd[1]/text()")[0]),
        "maxQuality" : int(tree.xpath("//dl/dt[text()='Maximum Quality']/following-sibling::dd[1]/text()")[0]),
    }

    for lang in LANGUAGES:
        tree = html.fromstring(pages[lang].result().text)
        recipe["name"][lang] = str(tree.xpath("//h2[contains(@class,'item_name')]/text()")[0])

    return recipe

def fetch(cls):
    recipes = []
    links = fetch_recipe_links(cls)
    for i in range(0, len(links)):
        print("\rFetching {0} recipe... {1} of {2}".format(cls, i+1, len(links)), end="")
        recipes.append(fetch_recipe(links[i]))
    print("\rFetching {0} recipes... done.".format(cls))
    return recipes

def scrape_to_file(cls):
    recipes = fetch(cls)
    with open("out/" + cls + ".json", "wt") as db_file:
        json.dump(recipes, db_file, indent="  ")
    print("Wrote {0} recipes for {1}".format(len(recipes), cls))

def main():
    for cls in CLASSES:
        scrape_to_file(cls)
    executor.shutdown()

if __name__ == '__main__':
    main()

@@ -33,27 +33,8 @@
 port
- urljoin, urlparse,
  url
@@ -141,20 +141,27 @@
  html%0A%0A%0A
-BASE
+RECIPE_LIST
 _URL = %22
@@ -231,17 +231,54 @@
 LANG
-UAGES = %7B
+_HOSTS = %7B%0A    %22en%22: %22na.finalfantasyxiv.com%22,
 %0A   
@@ -769,26 +769,16 @@
 ks_page(
-base_url, 
 r):%0A    
@@ -876,68 +876,27 @@
 s = 
-%5Burljoin(base_url, str(rel_link)) for rel_link in rel_links%5D
+map(str, rel_links)
 %0A   
@@ -1061,17 +1061,16 @@
  total%0A%0A
-%0A
 def fetc
@@ -1268,26 +1268,16 @@
 ks_page(
-BASE_URL, 
 session.
@@ -1284,12 +1284,19 @@
 get(
-BASE
+RECIPE_LIST
 _URL
@@ -1862,188 +1862,110 @@
 def 
-switch_url_host(url, host):%0A    parsed_url = urlparse(url)%0A    return urlunparse((parsed_url.scheme, host, parsed_url.path, parsed_url.params, parsed_url.query, parsed_url.fragment
+make_recipe_url(lang, rel_link):%0A    return urlunparse((%22http%22, LANG_HOSTS%5Blang%5D, rel_link, %22%22, %22%22, %22%22
 ))%0A%0A
@@ -1981,19 +1981,24 @@
 _recipe(
-url
+rel_link
 ):%0A    p
@@ -2005,20 +2005,20 @@
 ages = %7B
-%22en%22
+lang
 : execut
@@ -2044,149 +2044,64 @@
 et, 
-url)%7D%0A%0A    for lang in LANGUAGES:%0A        url = switch_url_host(url, LANGUAGES%5Blang%5D)%0A        pages%5Blang%5D = executor.submit(session.get, url)
+make_recipe_url(lang, rel_link)) for lang in LANG_HOSTS%7D
 %0A%0A  
@@ -2189,79 +2189,8 @@
  : %7B
- %22en%22: str(tree.xpath(%22//h2%5Bcontains(@class,'item_name')%5D/text()%22)%5B0%5D) 
 %7D,%0A 
@@ -2627,12 +2627,13 @@
 LANG
-UAGE
+_HOST
 S:%0A 

Normalize fetching of recipe pages and name extraction.