#!/usr/bin/env python
import requests
import hashlib
import argparse
import json
from termcolor import colored
from tabulate import tabulate
from urllib.parse import urlparse

"""
Script to detect the presence of dynamic rendering engines.

Based on the following blog post: https://r2c.dev/blog/2020/exploiting-dynamic-rendering-engines-to-take-control-of-web-apps/

Dependencies:
    pip3 install requests termcolor tabulate
"""

# Config
## Disable TLS warning when validation is disabled when requests is used
requests.packages.urllib3.disable_warnings(requests.packages.urllib3.exceptions.InsecureRequestWarning)

# Constants 
## Define proxy to debug request sent using requests
PROXIES = { }
BASELINE_UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:85.0) Gecko/20100101 Firefox/85.0"

class ResponseData:
    def __init__(self, response, user_agent_pattern):
        self.user_agent_pattern = user_agent_pattern
        self.return_code = response.status_code
        self.body_length = len(response.text)
        self.body_content_hash = hashlib.sha1(response.text.encode("utf-8")).hexdigest()
        self.has_scripts_tag = ("</script>" in response.text.lower())
        content = ""
        for header in response.headers:
            if header.lower() not in ["date", "set-cookie", "expires", "etag", "last-modified", "content-length"]:
                content += f"{header}={response.headers[header]}"
        self.headers_hash = hashlib.sha1(content.encode("utf-8")).hexdigest()

def test_ua(full_url, user_agent, user_agent_pattern):
    h = {"User-Agent": user_agent}
    try:
        response = requests.get(full_url, verify=False, proxies=PROXIES, allow_redirects=True, headers=h, timeout=30)
        return ResponseData(response, user_agent_pattern)
    except Exception as e:
        print(colored(f"[!] Error for User-Agent '{user_agent}': {e}", "red"))
        return None

def main(full_url, user_agent_list, user_agent_total):
    cases_result = []
    print(colored(f"[+] Execution context:", "magenta"))
    print(f"Full URL    : {full_url}")
    print(f"Proxy       : {PROXIES}")
    print(f"UA total    : {user_agent_total}")
    print(colored(f"[+] Send requests for the different User Agents...", "magenta"))
    responses = []
    progress = 0
    baseline_response = test_ua(full_url, BASELINE_UA, "Baseline")
    for user_agent_pattern in user_agent_list:
        print(f"\r{progress}/{user_agent_total}", end="", flush=True)
        responses.append(test_ua(full_url, user_agent_list[user_agent_pattern], user_agent_pattern))
        progress += 1
    print(colored(f"\n[+] Results - Show only responses without script tags except the first line that is the baseline i.e. normal request using desktop browser UA:", "magenta"))
    data = []
    data.append(["Type UA", "Body has script tags", "HTTP Code", "Body length", "Body hash", "Headers hash"])
    data.append([baseline_response.user_agent_pattern, baseline_response.has_scripts_tag, baseline_response.return_code, baseline_response.body_length, baseline_response.body_content_hash, baseline_response.headers_hash])
    for response in responses:
        if response is not None and not response.has_scripts_tag:
            data.append([response.user_agent_pattern, response.has_scripts_tag, response.return_code, response.body_length, response.body_content_hash, response.headers_hash])
    print(tabulate(data, headers="firstrow", tablefmt="github"))

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Detect the presence of dynamic rendering engines.")
    required_params = parser.add_argument_group("required named arguments")
    required_params.add_argument("-b", action="store", dest="full_url", help="Base URL (ex: 'https://righettod.eu').", required=True)    
    parser.add_argument("-x", action="store", dest="proxy", help="Proxy to use for all probe requests (ex: 'http://127.0.0.1:8080', default to no proxy).", required=False, default=None) 
    parser.add_argument("-f", action="store", dest="ua_json_file", help="User Agent database JSON file from 'https://github.com/monperrus/crawler-user-agents' repository.", required=False, default="/tools/crawler-user-agents/crawler-user-agents.json") 
    args = parser.parse_args()  
    if args.proxy is not None:
        PROXIES = { "http": args.proxy, "https": args.proxy } 
    user_agent_list = {}
    user_agent_total = 0
    with open(args.ua_json_file, "r", encoding="utf-8") as f:
        json_data = json.loads(f.read())
    for data in json_data:
        basename = data["pattern"]
        i = 0
        for ua in data["instances"]:
            user_agent_list[f"{basename}[{i}]"] = ua
            i += 1
            user_agent_total += 1
    main(args.full_url, user_agent_list, user_agent_total)
