# -*- coding: utf-8 -*-
# -------------------------------------------------------------------------------
# Name:         sfp_websvr
# Purpose:      SpiderFoot plug-in for scanning retreived content by other
#               modules (such as sfp_spider) and identifying web servers used
#
# Author:      Steve Micallef <steve@binarypool.com>
#
# Created:     06/04/2012
# Copyright:   (c) Steve Micallef 2012
# Licence:     GPL
# -------------------------------------------------------------------------------

import json
from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent


class sfp_websvr(SpiderFootPlugin):
    """Web Server:Footprint,Investigate:Content Analysis::Obtain web server banners to identify versions of web servers being used."""

    # Default options
    opts = {}

    results = dict()

    def setup(self, sfc, userOpts=dict()):
        self.sf = sfc
        self.results = dict()
        self.__dataSource__ = "Target Website"

        for opt in userOpts.keys():
            self.opts[opt] = userOpts[opt]

    # What events is this module interested in for input
    def watchedEvents(self):
        return ["WEBSERVER_HTTPHEADERS"]

    # What events this module produces
    # This is to support the end user in selecting modules based on events
    # produced.
    def producedEvents(self):
        return ["WEBSERVER_BANNER", "WEBSERVER_TECHNOLOGY"]

    # Handle events sent to this module
    def handleEvent(self, event):
        eventName = event.eventType
        srcModuleName = event.module
        eventData = event.data
        parentEvent = event.sourceEvent
        eventSource = event.sourceEvent.data

        self.sf.debug("Received event, " + eventName + ", from " + srcModuleName)
        if eventSource in self.results:
            return None
        else:
            self.results[eventSource] = True

        if not self.getTarget().matches(self.sf.urlFQDN(eventSource)):
            self.sf.debug("Not collecting web server information for external sites.")
            return None

        try:
            jdata = json.loads(eventData)
            if jdata == None:
                return None
        except BaseException as e:
            self.sf.error("Received HTTP headers from another module in an unexpected format.", False)
            return None

        # Could apply some smarts here, for instance looking for certain
        # banners and therefore classifying them further (type and version,
        # possibly OS. This could also trigger additional tests, such as 404s
        # and other errors to see what the header looks like.
        if 'server' in jdata:
            evt = SpiderFootEvent("WEBSERVER_BANNER", jdata['server'],
                                  self.__name__, parentEvent)
            self.notifyListeners(evt)

            self.sf.info("Found web server: " + jdata['server'] + " (" + eventSource + ")")

        if 'x-powered-by' in jdata:
            evt = SpiderFootEvent("WEBSERVER_TECHNOLOGY", jdata['x-powered-by'],
                                  self.__name__, parentEvent)
            self.notifyListeners(evt)
            return None

        tech = None
        if 'set-cookie'in jdata and 'PHPSESS' in jdata['set-cookie']:
            tech = "PHP"

        if 'set-cookie' in jdata and 'JSESSIONID' in jdata['set-cookie']:
            tech = "Java/JSP"

        if 'set-cookie' in jdata and 'ASP.NET' in jdata['set-cookie']:
            tech = "ASP.NET"

        if 'x-aspnet-version' in jdata:
            tech = "ASP.NET"

        if tech is not None and '.jsp' in eventSource:
            tech = "Java/JSP"

        if tech is not None and '.php' in eventSource:
            tech = "PHP"

        if tech is not None:
            evt = SpiderFootEvent("WEBSERVER_TECHNOLOGY", tech, self.__name__, parentEvent)
            self.notifyListeners(evt)

# End of sfp_websvr class

@@ -1381,16 +1381,78 @@
 HNOLOGY%22
+,%0A                'LINKED_URL_INTERNAL', 'LINKED_URL_EXTERNAL'
 %5D%0A%0A    #
@@ -2367,32 +2367,1369 @@
    return None%0A%0A
+        # Check location header for linked URLs%0A        if 'location' in jdata:%0A            if jdata%5B'location'%5D.startswith('http://') or jdata%5B'location'%5D.startswith('https://'):%0A                if self.getTarget().matches(self.sf.urlFQDN(jdata%5B'location'%5D)):%0A                    evt = SpiderFootEvent('LINKED_URL_INTERNAL', jdata%5B'location'%5D, self.__name__, parentEvent)%0A                    self.notifyListeners(evt)%0A                else:%0A                    evt = SpiderFootEvent('LINKED_URL_EXTERNAL', jdata%5B'location'%5D, self.__name__, parentEvent)%0A                    self.notifyListeners(evt)%0A%0A        # Check CSP header for linked URLs%0A        if 'content-security-policy' in jdata:%0A            for directive in jdata%5B'content-security-policy'%5D.split(';'):%0A                for string in directive.split(' '):%0A                    if string.startswith('http://') or string.startswith('https://'):%0A                        if self.getTarget().matches(self.sf.urlFQDN(string)):%0A                            evt = SpiderFootEvent('LINKED_URL_INTERNAL', string, self.__name__, parentEvent)%0A                            self.notifyListeners(evt)%0A                        else:%0A                            evt = SpiderFootEvent('LINKED_URL_EXTERNAL', string, self.__name__, parentEvent)%0A                            self.notifyListeners(evt)%0A%0A
         # Could 

Add URL extraction for Location and CSP headers